2024-11-24 02:46:04,049 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 02:46:04,064 main DEBUG Took 0.012455 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 02:46:04,065 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 02:46:04,065 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 02:46:04,066 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 02:46:04,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,077 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 02:46:04,093 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,095 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,097 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,098 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,099 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,100 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,101 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,102 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,103 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,103 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,104 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,105 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,106 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,106 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,107 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,108 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,108 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:04,109 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,109 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 02:46:04,111 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:04,113 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 02:46:04,115 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 02:46:04,115 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 02:46:04,117 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 02:46:04,117 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 02:46:04,128 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 02:46:04,131 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 02:46:04,133 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 02:46:04,133 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 02:46:04,134 main DEBUG createAppenders(={Console}) 2024-11-24 02:46:04,135 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-24 02:46:04,135 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 02:46:04,135 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-24 02:46:04,136 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 02:46:04,137 main DEBUG OutputStream closed 2024-11-24 02:46:04,137 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 02:46:04,137 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 02:46:04,138 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-24 02:46:04,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 02:46:04,226 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 02:46:04,227 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 02:46:04,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 02:46:04,229 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 02:46:04,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 02:46:04,230 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 02:46:04,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 02:46:04,231 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 02:46:04,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 02:46:04,232 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 02:46:04,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 02:46:04,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 02:46:04,233 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 02:46:04,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 02:46:04,234 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 02:46:04,235 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 02:46:04,236 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 02:46:04,239 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 02:46:04,239 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-24 02:46:04,240 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 02:46:04,241 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-24T02:46:04,513 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a 2024-11-24 02:46:04,515 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 02:46:04,515 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T02:46:04,524 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-24T02:46:04,544 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T02:46:04,561 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545, deleteOnExit=true 2024-11-24T02:46:04,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T02:46:04,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/test.cache.data in system properties and HBase conf 2024-11-24T02:46:04,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T02:46:04,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/hadoop.log.dir in system properties and HBase conf 2024-11-24T02:46:04,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T02:46:04,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T02:46:04,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T02:46:04,646 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T02:46:04,758 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T02:46:04,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:46:04,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T02:46:04,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T02:46:04,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:46:04,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T02:46:04,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T02:46:04,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T02:46:04,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:46:04,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T02:46:04,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/nfs.dump.dir in system properties and HBase conf 2024-11-24T02:46:04,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/java.io.tmpdir in system properties and HBase conf 2024-11-24T02:46:04,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T02:46:04,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T02:46:04,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T02:46:05,762 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T02:46:05,828 INFO [Time-limited test {}] log.Log(170): Logging initialized @2483ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T02:46:05,892 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:46:05,951 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:46:05,971 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:46:05,971 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:46:05,972 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T02:46:05,983 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:46:05,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:46:05,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T02:46:06,156 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/java.io.tmpdir/jetty-localhost-40005-hadoop-hdfs-3_4_1-tests_jar-_-any-5729924325373949969/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T02:46:06,162 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:40005} 2024-11-24T02:46:06,163 INFO [Time-limited test {}] server.Server(415): Started @2818ms 2024-11-24T02:46:06,740 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T02:46:06,747 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T02:46:06,748 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T02:46:06,749 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T02:46:06,749 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T02:46:06,750 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/hadoop.log.dir/,AVAILABLE} 2024-11-24T02:46:06,750 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T02:46:06,851 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/java.io.tmpdir/jetty-localhost-33311-hadoop-hdfs-3_4_1-tests_jar-_-any-8984490794710077675/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T02:46:06,852 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:33311} 2024-11-24T02:46:06,852 INFO [Time-limited test {}] server.Server(415): Started @3507ms 2024-11-24T02:46:06,901 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T02:46:07,950 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545/data/data2/current/BP-79769874-172.17.0.2-1732416365305/current, will proceed with Du for space computation calculation, 2024-11-24T02:46:07,950 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545/data/data1/current/BP-79769874-172.17.0.2-1732416365305/current, will proceed with Du for space computation calculation, 2024-11-24T02:46:07,997 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T02:46:08,050 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe04775548e378a55 with lease ID 0xf7a75c6252e2d119: Processing first storage report for DS-826f8768-4810-4b9b-aa12-a05a86c2105f from datanode DatanodeRegistration(127.0.0.1:36769, datanodeUuid=ef9ddf6b-c708-403b-baaa-8ef179148346, infoPort=44283, infoSecurePort=0, ipcPort=41965, storageInfo=lv=-57;cid=testClusterID;nsid=2004851247;c=1732416365305) 2024-11-24T02:46:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe04775548e378a55 with lease ID 0xf7a75c6252e2d119: from storage DS-826f8768-4810-4b9b-aa12-a05a86c2105f node DatanodeRegistration(127.0.0.1:36769, datanodeUuid=ef9ddf6b-c708-403b-baaa-8ef179148346, infoPort=44283, infoSecurePort=0, ipcPort=41965, storageInfo=lv=-57;cid=testClusterID;nsid=2004851247;c=1732416365305), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-24T02:46:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe04775548e378a55 with lease ID 0xf7a75c6252e2d119: Processing first storage report for DS-8f97f04f-5dda-4d86-892c-663fed745d41 from datanode DatanodeRegistration(127.0.0.1:36769, datanodeUuid=ef9ddf6b-c708-403b-baaa-8ef179148346, infoPort=44283, infoSecurePort=0, ipcPort=41965, storageInfo=lv=-57;cid=testClusterID;nsid=2004851247;c=1732416365305) 2024-11-24T02:46:08,054 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe04775548e378a55 with lease ID 0xf7a75c6252e2d119: from storage DS-8f97f04f-5dda-4d86-892c-663fed745d41 node DatanodeRegistration(127.0.0.1:36769, datanodeUuid=ef9ddf6b-c708-403b-baaa-8ef179148346, infoPort=44283, infoSecurePort=0, ipcPort=41965, storageInfo=lv=-57;cid=testClusterID;nsid=2004851247;c=1732416365305), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T02:46:08,109 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a 2024-11-24T02:46:08,172 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545/zookeeper_0, clientPort=55039, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T02:46:08,182 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55039 2024-11-24T02:46:08,192 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:46:08,195 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:46:08,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741825_1001 (size=7) 2024-11-24T02:46:08,785 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 with version=8 2024-11-24T02:46:08,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/hbase-staging 2024-11-24T02:46:08,866 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T02:46:09,188 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c69a60bd8f6:0 server-side Connection retries=45 2024-11-24T02:46:09,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:46:09,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T02:46:09,203 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T02:46:09,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:46:09,203 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T02:46:09,326 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T02:46:09,377 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T02:46:09,385 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T02:46:09,389 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T02:46:09,409 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19664 (auto-detected) 2024-11-24T02:46:09,410 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T02:46:09,426 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43749 2024-11-24T02:46:09,445 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43749 connecting to ZooKeeper ensemble=127.0.0.1:55039 2024-11-24T02:46:09,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:437490x0, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T02:46:09,541 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43749-0x1016abbe9c70000 connected 2024-11-24T02:46:09,705 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:46:09,710 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:46:09,724 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:46:09,728 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3, hbase.cluster.distributed=false 2024-11-24T02:46:09,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T02:46:09,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43749 2024-11-24T02:46:09,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43749 2024-11-24T02:46:09,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43749 2024-11-24T02:46:09,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43749 2024-11-24T02:46:09,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43749 2024-11-24T02:46:09,858 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c69a60bd8f6:0 server-side Connection retries=45 2024-11-24T02:46:09,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:46:09,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T02:46:09,860 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T02:46:09,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T02:46:09,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T02:46:09,863 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T02:46:09,867 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T02:46:09,868 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39461 2024-11-24T02:46:09,870 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39461 connecting to ZooKeeper ensemble=127.0.0.1:55039 2024-11-24T02:46:09,871 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:46:09,874 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:46:09,898 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394610x0, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T02:46:09,899 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39461-0x1016abbe9c70001 connected 2024-11-24T02:46:09,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:46:09,903 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T02:46:09,910 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T02:46:09,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T02:46:09,918 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T02:46:09,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39461 2024-11-24T02:46:09,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39461 2024-11-24T02:46:09,920 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39461 2024-11-24T02:46:09,921 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39461 2024-11-24T02:46:09,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39461 2024-11-24T02:46:09,943 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c69a60bd8f6:43749 2024-11-24T02:46:09,944 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:09,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:46:09,961 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:46:09,963 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:09,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:09,997 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T02:46:09,997 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:09,998 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T02:46:09,999 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c69a60bd8f6,43749,1732416369043 from backup master directory 2024-11-24T02:46:10,013 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:46:10,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:10,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T02:46:10,014 WARN [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T02:46:10,014 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:10,016 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T02:46:10,018 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T02:46:10,072 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/hbase.id] with ID: 4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc 2024-11-24T02:46:10,073 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/.tmp/hbase.id 2024-11-24T02:46:10,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741826_1002 (size=42) 2024-11-24T02:46:10,490 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/.tmp/hbase.id]:[hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/hbase.id] 2024-11-24T02:46:10,539 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T02:46:10,543 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T02:46:10,561 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-24T02:46:10,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:10,613 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:10,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741827_1003 (size=196) 2024-11-24T02:46:11,055 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:46:11,057 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T02:46:11,071 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:11,074 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:46:11,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741828_1004 (size=1189) 2024-11-24T02:46:11,527 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store 2024-11-24T02:46:11,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741829_1005 (size=34) 2024-11-24T02:46:11,549 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T02:46:11,552 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:11,553 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T02:46:11,553 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:46:11,554 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:46:11,555 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T02:46:11,555 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:46:11,555 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:46:11,556 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732416371553Disabling compacts and flushes for region at 1732416371553Disabling writes for close at 1732416371555 (+2 ms)Writing region close event to WAL at 1732416371555Closed at 1732416371555 2024-11-24T02:46:11,558 WARN [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/.initializing 2024-11-24T02:46:11,559 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/WALs/7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:11,567 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T02:46:11,580 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c69a60bd8f6%2C43749%2C1732416369043, suffix=, logDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/WALs/7c69a60bd8f6,43749,1732416369043, archiveDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/oldWALs, maxLogs=10 2024-11-24T02:46:11,601 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/WALs/7c69a60bd8f6,43749,1732416369043/7c69a60bd8f6%2C43749%2C1732416369043.1732416371584, exclude list is [], retry=0 2024-11-24T02:46:11,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36769,DS-826f8768-4810-4b9b-aa12-a05a86c2105f,DISK] 2024-11-24T02:46:11,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T02:46:11,648 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/WALs/7c69a60bd8f6,43749,1732416369043/7c69a60bd8f6%2C43749%2C1732416369043.1732416371584 2024-11-24T02:46:11,649 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44283:44283)] 2024-11-24T02:46:11,649 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:46:11,650 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:11,653 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,653 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T02:46:11,712 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:11,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:11,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T02:46:11,718 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:11,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:11,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T02:46:11,722 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:11,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:11,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T02:46:11,726 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:11,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:11,728 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,731 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,732 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,737 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,737 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,741 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T02:46:11,744 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T02:46:11,748 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:46:11,750 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71837592, jitterRate=0.07046353816986084}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T02:46:11,756 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732416371664Initializing all the Stores at 1732416371666 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416371666Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416371667 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416371667Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416371667Cleaning up temporary data from old regions at 1732416371737 (+70 ms)Region opened successfully at 1732416371756 (+19 ms) 2024-11-24T02:46:11,758 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T02:46:11,786 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d03a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c69a60bd8f6/172.17.0.2:0 2024-11-24T02:46:11,812 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T02:46:11,822 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T02:46:11,822 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T02:46:11,825 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T02:46:11,826 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T02:46:11,830 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-24T02:46:11,830 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T02:46:11,852 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T02:46:11,860 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T02:46:11,987 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T02:46:11,992 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T02:46:11,995 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T02:46:12,084 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T02:46:12,089 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T02:46:12,097 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T02:46:12,144 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T02:46:12,147 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T02:46:12,160 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T02:46:12,184 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T02:46:12,197 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T02:46:12,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T02:46:12,208 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T02:46:12,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:12,208 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:12,211 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c69a60bd8f6,43749,1732416369043, sessionid=0x1016abbe9c70000, setting cluster-up flag (Was=false) 2024-11-24T02:46:12,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:12,244 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:12,276 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T02:46:12,280 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:12,308 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:12,308 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:12,339 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T02:46:12,343 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:12,352 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T02:46:12,431 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T02:46:12,434 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(746): ClusterId : 4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc 2024-11-24T02:46:12,437 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T02:46:12,442 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T02:46:12,448 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T02:46:12,452 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T02:46:12,452 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T02:46:12,454 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c69a60bd8f6,43749,1732416369043 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T02:46:12,461 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T02:46:12,462 DEBUG [RS:0;7c69a60bd8f6:39461 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d19b12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c69a60bd8f6/172.17.0.2:0 2024-11-24T02:46:12,462 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:46:12,463 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:46:12,463 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:46:12,463 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=5, maxPoolSize=5 2024-11-24T02:46:12,463 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c69a60bd8f6:0, corePoolSize=10, maxPoolSize=10 2024-11-24T02:46:12,463 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,464 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T02:46:12,464 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,468 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732416402468 2024-11-24T02:46:12,469 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:46:12,469 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T02:46:12,470 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T02:46:12,471 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T02:46:12,475 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:12,475 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T02:46:12,475 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T02:46:12,476 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T02:46:12,476 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T02:46:12,476 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T02:46:12,477 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,480 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c69a60bd8f6:39461 2024-11-24T02:46:12,481 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T02:46:12,482 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T02:46:12,482 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T02:46:12,484 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T02:46:12,484 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T02:46:12,484 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T02:46:12,486 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c69a60bd8f6,43749,1732416369043 with port=39461, startcode=1732416369827 2024-11-24T02:46:12,489 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T02:46:12,489 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T02:46:12,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741831_1007 (size=1321) 2024-11-24T02:46:12,491 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.large.0-1732416372490,5,FailOnTimeoutGroup] 2024-11-24T02:46:12,495 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.small.0-1732416372491,5,FailOnTimeoutGroup] 2024-11-24T02:46:12,495 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,496 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T02:46:12,497 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,497 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,498 DEBUG [RS:0;7c69a60bd8f6:39461 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T02:46:12,536 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T02:46:12,544 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43749 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:12,547 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43749 {}] master.ServerManager(517): Registering regionserver=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:12,562 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:46:12,563 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40051 2024-11-24T02:46:12,563 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T02:46:12,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T02:46:12,576 DEBUG [RS:0;7c69a60bd8f6:39461 {}] zookeeper.ZKUtil(111): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:12,577 WARN [RS:0;7c69a60bd8f6:39461 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T02:46:12,577 INFO [RS:0;7c69a60bd8f6:39461 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:46:12,577 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/WALs/7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:12,579 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c69a60bd8f6,39461,1732416369827] 2024-11-24T02:46:12,603 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T02:46:12,618 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T02:46:12,622 INFO [RS:0;7c69a60bd8f6:39461 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T02:46:12,622 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,623 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T02:46:12,628 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T02:46:12,630 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,630 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,630 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,630 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,630 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,631 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,631 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c69a60bd8f6:0, corePoolSize=2, maxPoolSize=2 2024-11-24T02:46:12,631 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,631 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,631 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,631 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,632 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,632 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c69a60bd8f6:0, corePoolSize=1, maxPoolSize=1 2024-11-24T02:46:12,632 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c69a60bd8f6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T02:46:12,632 DEBUG [RS:0;7c69a60bd8f6:39461 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0, corePoolSize=3, maxPoolSize=3 2024-11-24T02:46:12,633 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,634 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,634 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,634 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,634 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,634 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,39461,1732416369827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T02:46:12,653 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T02:46:12,655 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,39461,1732416369827-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,656 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,656 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.Replication(171): 7c69a60bd8f6,39461,1732416369827 started 2024-11-24T02:46:12,674 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:12,674 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1482): Serving as 7c69a60bd8f6,39461,1732416369827, RpcServer on 7c69a60bd8f6/172.17.0.2:39461, sessionid=0x1016abbe9c70001 2024-11-24T02:46:12,675 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T02:46:12,675 DEBUG [RS:0;7c69a60bd8f6:39461 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:12,675 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c69a60bd8f6,39461,1732416369827' 2024-11-24T02:46:12,675 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T02:46:12,676 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T02:46:12,677 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T02:46:12,677 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T02:46:12,678 DEBUG [RS:0;7c69a60bd8f6:39461 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:12,678 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c69a60bd8f6,39461,1732416369827' 2024-11-24T02:46:12,678 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T02:46:12,679 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T02:46:12,679 DEBUG [RS:0;7c69a60bd8f6:39461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T02:46:12,679 INFO [RS:0;7c69a60bd8f6:39461 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T02:46:12,679 INFO [RS:0;7c69a60bd8f6:39461 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T02:46:12,788 INFO [RS:0;7c69a60bd8f6:39461 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T02:46:12,792 INFO [RS:0;7c69a60bd8f6:39461 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c69a60bd8f6%2C39461%2C1732416369827, suffix=, logDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/WALs/7c69a60bd8f6,39461,1732416369827, archiveDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/oldWALs, maxLogs=32 2024-11-24T02:46:12,808 DEBUG [RS:0;7c69a60bd8f6:39461 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/WALs/7c69a60bd8f6,39461,1732416369827/7c69a60bd8f6%2C39461%2C1732416369827.1732416372794, exclude list is [], retry=0 2024-11-24T02:46:12,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36769,DS-826f8768-4810-4b9b-aa12-a05a86c2105f,DISK] 2024-11-24T02:46:12,815 INFO [RS:0;7c69a60bd8f6:39461 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/WALs/7c69a60bd8f6,39461,1732416369827/7c69a60bd8f6%2C39461%2C1732416369827.1732416372794 2024-11-24T02:46:12,816 DEBUG [RS:0;7c69a60bd8f6:39461 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44283:44283)] 2024-11-24T02:46:12,894 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T02:46:12,894 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:46:12,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741833_1009 (size=32) 2024-11-24T02:46:13,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:13,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T02:46:13,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T02:46:13,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T02:46:13,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T02:46:13,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T02:46:13,320 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T02:46:13,320 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T02:46:13,326 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T02:46:13,326 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,328 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T02:46:13,330 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740 2024-11-24T02:46:13,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740 2024-11-24T02:46:13,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T02:46:13,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T02:46:13,336 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:46:13,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T02:46:13,345 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:46:13,347 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71498798, jitterRate=0.06541511416435242}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:46:13,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732416373306Initializing all the Stores at 1732416373308 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416373308Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416373308Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416373308Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416373308Cleaning up temporary data from old regions at 1732416373335 (+27 ms)Region opened successfully at 1732416373349 (+14 ms) 2024-11-24T02:46:13,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T02:46:13,350 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T02:46:13,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T02:46:13,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T02:46:13,350 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T02:46:13,351 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T02:46:13,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732416373349Disabling compacts and flushes for region at 1732416373349Disabling writes for close at 1732416373350 (+1 ms)Writing region close event to WAL at 1732416373351 (+1 ms)Closed at 1732416373351 2024-11-24T02:46:13,356 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:46:13,356 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T02:46:13,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T02:46:13,373 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T02:46:13,377 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T02:46:13,530 DEBUG [7c69a60bd8f6:43749 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T02:46:13,542 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:13,548 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c69a60bd8f6,39461,1732416369827, state=OPENING 2024-11-24T02:46:13,618 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T02:46:13,629 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:13,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:46:13,630 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:46:13,630 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:46:13,632 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T02:46:13,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:46:13,813 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T02:46:13,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60493, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T02:46:13,831 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T02:46:13,831 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T02:46:13,837 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T02:46:13,842 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c69a60bd8f6%2C39461%2C1732416369827.meta, suffix=.meta, logDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/WALs/7c69a60bd8f6,39461,1732416369827, archiveDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/oldWALs, maxLogs=32 2024-11-24T02:46:13,865 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/WALs/7c69a60bd8f6,39461,1732416369827/7c69a60bd8f6%2C39461%2C1732416369827.meta.1732416373844.meta, exclude list is [], retry=0 2024-11-24T02:46:13,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36769,DS-826f8768-4810-4b9b-aa12-a05a86c2105f,DISK] 2024-11-24T02:46:13,873 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/WALs/7c69a60bd8f6,39461,1732416369827/7c69a60bd8f6%2C39461%2C1732416369827.meta.1732416373844.meta 2024-11-24T02:46:13,873 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44283:44283)] 2024-11-24T02:46:13,874 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:46:13,876 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T02:46:13,880 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T02:46:13,886 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T02:46:13,891 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T02:46:13,892 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:13,892 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T02:46:13,892 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T02:46:13,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T02:46:13,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T02:46:13,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T02:46:13,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T02:46:13,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T02:46:13,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T02:46:13,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T02:46:13,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T02:46:13,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:13,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T02:46:13,912 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T02:46:13,914 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740 2024-11-24T02:46:13,918 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740 2024-11-24T02:46:13,921 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T02:46:13,922 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T02:46:13,923 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:46:13,932 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T02:46:13,935 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67347159, jitterRate=0.0035508722066879272}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:46:13,935 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T02:46:13,937 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732416373893Writing region info on filesystem at 1732416373893Initializing all the Stores at 1732416373895 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416373895Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416373896 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416373896Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732416373896Cleaning up temporary data from old regions at 1732416373922 (+26 ms)Running coprocessor post-open hooks at 1732416373935 (+13 ms)Region opened successfully at 1732416373936 (+1 ms) 2024-11-24T02:46:13,945 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732416373804 2024-11-24T02:46:13,958 DEBUG [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T02:46:13,959 INFO [RS_OPEN_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T02:46:13,961 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:13,964 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c69a60bd8f6,39461,1732416369827, state=OPEN 2024-11-24T02:46:14,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T02:46:14,082 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T02:46:14,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:46:14,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T02:46:14,083 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:14,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T02:46:14,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c69a60bd8f6,39461,1732416369827 in 449 msec 2024-11-24T02:46:14,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T02:46:14,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 730 msec 2024-11-24T02:46:14,101 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T02:46:14,101 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T02:46:14,122 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:14,124 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:14,150 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:14,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:14,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7880 sec 2024-11-24T02:46:14,177 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732416374177, completionTime=-1 2024-11-24T02:46:14,179 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T02:46:14,180 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T02:46:14,209 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T02:46:14,209 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732416434209 2024-11-24T02:46:14,210 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732416494210 2024-11-24T02:46:14,210 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-24T02:46:14,213 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,43749,1732416369043-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:14,213 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,43749,1732416369043-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:14,213 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,43749,1732416369043-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:14,215 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c69a60bd8f6:43749, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:14,215 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:14,216 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:14,222 DEBUG [master/7c69a60bd8f6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T02:46:14,251 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.237sec 2024-11-24T02:46:14,252 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T02:46:14,254 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T02:46:14,255 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T02:46:14,255 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T02:46:14,256 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T02:46:14,257 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,43749,1732416369043-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T02:46:14,257 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,43749,1732416369043-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T02:46:14,265 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T02:46:14,266 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T02:46:14,268 INFO [master/7c69a60bd8f6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c69a60bd8f6,43749,1732416369043-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T02:46:14,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:14,346 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T02:46:14,346 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T02:46:14,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:14,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:14,362 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2479a655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:14,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:14,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:14,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:14,377 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:14,381 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:14,382 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:14,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:14,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:14,442 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1135, ProcessCount=11, AvailableMemoryMB=11786 2024-11-24T02:46:14,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:46:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:14,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4c1af104 2024-11-24T02:46:14,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:46:14,469 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:46:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:46:14,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:46:14,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:14,494 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:46:14,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-24T02:46:14,497 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:14,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:46:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:46:14,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741835_1011 (size=959) 2024-11-24T02:46:14,519 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:46:14,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741836_1012 (size=53) 2024-11-24T02:46:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:46:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:46:14,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:14,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 129703e293b1a78847241fc258ac8730, disabling compactions & flushes 2024-11-24T02:46:14,930 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:14,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:14,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. after waiting 0 ms 2024-11-24T02:46:14,930 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:14,930 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:14,931 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 129703e293b1a78847241fc258ac8730: Waiting for close lock at 1732416374930Disabling compacts and flushes for region at 1732416374930Disabling writes for close at 1732416374930Writing region close event to WAL at 1732416374930Closed at 1732416374930 2024-11-24T02:46:14,933 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:46:14,940 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416374934"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416374934"}]},"ts":"1732416374934"} 2024-11-24T02:46:14,944 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:46:14,947 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:46:14,949 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416374947"}]},"ts":"1732416374947"} 2024-11-24T02:46:14,954 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:46:14,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, ASSIGN}] 2024-11-24T02:46:14,958 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, ASSIGN 2024-11-24T02:46:14,961 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=false 2024-11-24T02:46:15,112 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=129703e293b1a78847241fc258ac8730, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:15,117 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, ASSIGN because future has completed 2024-11-24T02:46:15,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:46:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:46:15,283 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:15,283 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:46:15,284 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,284 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:15,284 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,284 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,287 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,291 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 129703e293b1a78847241fc258ac8730 columnFamilyName A 2024-11-24T02:46:15,291 DEBUG [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:15,292 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] regionserver.HStore(327): Store=129703e293b1a78847241fc258ac8730/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:15,292 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,295 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 129703e293b1a78847241fc258ac8730 columnFamilyName B 2024-11-24T02:46:15,295 DEBUG [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:15,296 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] regionserver.HStore(327): Store=129703e293b1a78847241fc258ac8730/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:15,296 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,298 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 129703e293b1a78847241fc258ac8730 columnFamilyName C 2024-11-24T02:46:15,298 DEBUG [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:15,299 INFO [StoreOpener-129703e293b1a78847241fc258ac8730-1 {}] regionserver.HStore(327): Store=129703e293b1a78847241fc258ac8730/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:15,299 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,301 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,302 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,303 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,303 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,304 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:46:15,306 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,309 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:46:15,310 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 129703e293b1a78847241fc258ac8730; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62072366, jitterRate=-0.07504966855049133}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:46:15,310 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,311 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 129703e293b1a78847241fc258ac8730: Running coprocessor pre-open hook at 1732416375285Writing region info on filesystem at 1732416375285Initializing all the Stores at 1732416375287 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416375287Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416375287Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416375287Cleaning up temporary data from old regions at 1732416375303 (+16 ms)Running coprocessor post-open hooks at 1732416375310 (+7 ms)Region opened successfully at 1732416375311 (+1 ms) 2024-11-24T02:46:15,313 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., pid=6, masterSystemTime=1732416375272 2024-11-24T02:46:15,316 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:15,317 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:15,318 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=129703e293b1a78847241fc258ac8730, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:15,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:46:15,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T02:46:15,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 in 206 msec 2024-11-24T02:46:15,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T02:46:15,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, ASSIGN in 374 msec 2024-11-24T02:46:15,336 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:46:15,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416375337"}]},"ts":"1732416375337"} 2024-11-24T02:46:15,340 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:46:15,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:46:15,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 860 msec 2024-11-24T02:46:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T02:46:15,648 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:15,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,656 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d463185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,660 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f0fa42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,662 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,671 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:46:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64ba8396 2024-11-24T02:46:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:46:15,679 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:46:15,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,687 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470128a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,692 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174557e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,699 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,703 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,713 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,720 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@483a22c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,726 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32afd8f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6af04b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,736 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5eb702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,741 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f0b2d87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168b0e72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,750 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6396285d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,753 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439f6406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,757 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92f3d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,760 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:15,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:15,764 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cba810e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:15,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c8f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:15,774 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,774 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,774 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,777 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37136, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,781 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,783 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,783 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,788 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,800 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-24T02:46:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:46:15,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,807 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:15,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:15,809 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:15,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:15,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:15,812 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:15,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:15,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:46:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:46:15,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:15,914 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:15,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:16,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:46:16,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:16,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d17437b84b9345c4babaf0296a0342a2 is 50, key is test_row_0/A:col10/1732416375921/Put/seqid=0 2024-11-24T02:46:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:46:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416436130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416436122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416436115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741837_1013 (size=19021) 2024-11-24T02:46:16,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d17437b84b9345c4babaf0296a0342a2 2024-11-24T02:46:16,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416436156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416436159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:46:16,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/d100c27a2ac044adb0c6c7d7cc25e43b is 50, key is test_row_0/B:col10/1732416375921/Put/seqid=0 2024-11-24T02:46:16,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416436297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416436298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416436298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416436300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416436299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741838_1014 (size=12001) 2024-11-24T02:46:16,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/d100c27a2ac044adb0c6c7d7cc25e43b 2024-11-24T02:46:16,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/05c680fa246f4bd7ba65c5416c95da05 is 50, key is test_row_0/C:col10/1732416375921/Put/seqid=0 2024-11-24T02:46:16,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:46:16,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:46:16,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741839_1015 (size=12001) 2024-11-24T02:46:16,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416436517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416436517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416436519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416436528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416436537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:46:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:16,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:46:16,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:16,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416436828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/05c680fa246f4bd7ba65c5416c95da05 2024-11-24T02:46:16,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416436840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416436841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,846 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416436841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d17437b84b9345c4babaf0296a0342a2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d17437b84b9345c4babaf0296a0342a2 2024-11-24T02:46:16,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:16,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416436858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:16,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d17437b84b9345c4babaf0296a0342a2, entries=300, sequenceid=13, filesize=18.6 K 2024-11-24T02:46:16,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/d100c27a2ac044adb0c6c7d7cc25e43b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/d100c27a2ac044adb0c6c7d7cc25e43b 2024-11-24T02:46:16,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:46:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:16,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:16,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/d100c27a2ac044adb0c6c7d7cc25e43b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T02:46:16,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/05c680fa246f4bd7ba65c5416c95da05 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/05c680fa246f4bd7ba65c5416c95da05 2024-11-24T02:46:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:46:16,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/05c680fa246f4bd7ba65c5416c95da05, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T02:46:16,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 129703e293b1a78847241fc258ac8730 in 1051ms, sequenceid=13, compaction requested=false 2024-11-24T02:46:16,977 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T02:46:16,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:17,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T02:46:17,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:17,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:46:17,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/218b6114b3b6435fafe11ab3cfe27a99 is 50, key is test_row_0/A:col10/1732416376148/Put/seqid=0 2024-11-24T02:46:17,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741840_1016 (size=12001) 2024-11-24T02:46:17,086 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/218b6114b3b6435fafe11ab3cfe27a99 2024-11-24T02:46:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b7ccff740c3047b8b7351ed40de64b8c is 50, key is test_row_0/B:col10/1732416376148/Put/seqid=0 2024-11-24T02:46:17,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741841_1017 (size=12001) 2024-11-24T02:46:17,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b7ccff740c3047b8b7351ed40de64b8c 2024-11-24T02:46:17,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/fbaa3270e3f34f588570c7eceeffcf2b is 50, key is test_row_0/C:col10/1732416376148/Put/seqid=0 2024-11-24T02:46:17,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741842_1018 (size=12001) 2024-11-24T02:46:17,262 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/fbaa3270e3f34f588570c7eceeffcf2b 2024-11-24T02:46:17,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/218b6114b3b6435fafe11ab3cfe27a99 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/218b6114b3b6435fafe11ab3cfe27a99 2024-11-24T02:46:17,311 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/218b6114b3b6435fafe11ab3cfe27a99, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:46:17,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b7ccff740c3047b8b7351ed40de64b8c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7ccff740c3047b8b7351ed40de64b8c 2024-11-24T02:46:17,328 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7ccff740c3047b8b7351ed40de64b8c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:46:17,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/fbaa3270e3f34f588570c7eceeffcf2b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/fbaa3270e3f34f588570c7eceeffcf2b 2024-11-24T02:46:17,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:17,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/fbaa3270e3f34f588570c7eceeffcf2b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:46:17,354 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 129703e293b1a78847241fc258ac8730 in 317ms, sequenceid=37, compaction requested=false 2024-11-24T02:46:17,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-24T02:46:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-24T02:46:17,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T02:46:17,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5460 sec 2024-11-24T02:46:17,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5750 sec 2024-11-24T02:46:17,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:17,404 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:17,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:17,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7834cd6d507a4eeaa40d33d7fe8a158a is 50, key is test_row_0/A:col10/1732416377380/Put/seqid=0 2024-11-24T02:46:17,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741843_1019 (size=14341) 2024-11-24T02:46:17,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7834cd6d507a4eeaa40d33d7fe8a158a 2024-11-24T02:46:17,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/80350d3fb0fb4874bfe73563c8db3f1a is 50, key is test_row_0/B:col10/1732416377380/Put/seqid=0 2024-11-24T02:46:17,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741844_1020 (size=12001) 2024-11-24T02:46:17,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/80350d3fb0fb4874bfe73563c8db3f1a 2024-11-24T02:46:17,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/edee406d41f0446b983a82baa823b402 is 50, key is test_row_0/C:col10/1732416377380/Put/seqid=0 2024-11-24T02:46:17,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741845_1021 (size=12001) 2024-11-24T02:46:17,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/edee406d41f0446b983a82baa823b402 2024-11-24T02:46:17,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7834cd6d507a4eeaa40d33d7fe8a158a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7834cd6d507a4eeaa40d33d7fe8a158a 2024-11-24T02:46:17,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7834cd6d507a4eeaa40d33d7fe8a158a, entries=200, sequenceid=48, filesize=14.0 K 2024-11-24T02:46:17,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/80350d3fb0fb4874bfe73563c8db3f1a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/80350d3fb0fb4874bfe73563c8db3f1a 2024-11-24T02:46:17,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416437579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416437579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416437581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416437606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/80350d3fb0fb4874bfe73563c8db3f1a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T02:46:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/edee406d41f0446b983a82baa823b402 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/edee406d41f0446b983a82baa823b402 2024-11-24T02:46:17,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416437624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/edee406d41f0446b983a82baa823b402, entries=150, sequenceid=48, filesize=11.7 K 2024-11-24T02:46:17,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 129703e293b1a78847241fc258ac8730 in 248ms, sequenceid=48, compaction requested=true 2024-11-24T02:46:17,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:17,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:17,679 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:17,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:17,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:17,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:17,681 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:17,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:17,687 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:17,688 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:17,689 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:17,690 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:17,691 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/d100c27a2ac044adb0c6c7d7cc25e43b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7ccff740c3047b8b7351ed40de64b8c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/80350d3fb0fb4874bfe73563c8db3f1a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=35.2 K 2024-11-24T02:46:17,693 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d100c27a2ac044adb0c6c7d7cc25e43b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416375851 2024-11-24T02:46:17,693 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:17,693 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:17,694 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d17437b84b9345c4babaf0296a0342a2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/218b6114b3b6435fafe11ab3cfe27a99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7834cd6d507a4eeaa40d33d7fe8a158a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=44.3 K 2024-11-24T02:46:17,695 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ccff740c3047b8b7351ed40de64b8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416376123 2024-11-24T02:46:17,696 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d17437b84b9345c4babaf0296a0342a2, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416375798 2024-11-24T02:46:17,696 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 80350d3fb0fb4874bfe73563c8db3f1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416377380 2024-11-24T02:46:17,698 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218b6114b3b6435fafe11ab3cfe27a99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416376123 2024-11-24T02:46:17,699 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7834cd6d507a4eeaa40d33d7fe8a158a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416377379 2024-11-24T02:46:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:17,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:46:17,738 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:17,740 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/7a43a87368c844928aea94b74782bb57 is 50, key is test_row_0/B:col10/1732416377380/Put/seqid=0 2024-11-24T02:46:17,742 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:17,743 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/afa85aa9fe5d45e5b26014ef943c2055 is 50, key is test_row_0/A:col10/1732416377380/Put/seqid=0 2024-11-24T02:46:17,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/04c7743d0c2f4da18db106c4580d71cc is 50, key is test_row_0/A:col10/1732416377500/Put/seqid=0 2024-11-24T02:46:17,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416437751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416437758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416437758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416437759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416437767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741846_1022 (size=12104) 2024-11-24T02:46:17,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741847_1023 (size=12104) 2024-11-24T02:46:17,814 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/7a43a87368c844928aea94b74782bb57 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7a43a87368c844928aea94b74782bb57 2024-11-24T02:46:17,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741848_1024 (size=14341) 2024-11-24T02:46:17,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/04c7743d0c2f4da18db106c4580d71cc 2024-11-24T02:46:17,821 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/afa85aa9fe5d45e5b26014ef943c2055 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/afa85aa9fe5d45e5b26014ef943c2055 2024-11-24T02:46:17,852 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 7a43a87368c844928aea94b74782bb57(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:17,852 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:17,855 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416377681; duration=0sec 2024-11-24T02:46:17,856 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into afa85aa9fe5d45e5b26014ef943c2055(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:17,857 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:17,857 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416377656; duration=0sec 2024-11-24T02:46:17,857 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:17,858 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:17,858 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:17,859 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:17,859 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:17,864 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:17,864 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:17,864 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:17,864 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/05c680fa246f4bd7ba65c5416c95da05, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/fbaa3270e3f34f588570c7eceeffcf2b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/edee406d41f0446b983a82baa823b402] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=35.2 K 2024-11-24T02:46:17,866 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c680fa246f4bd7ba65c5416c95da05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416375851 2024-11-24T02:46:17,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/095da2814aec40eb879740514bba2f0e is 50, key is test_row_0/B:col10/1732416377500/Put/seqid=0 2024-11-24T02:46:17,869 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbaa3270e3f34f588570c7eceeffcf2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416376123 2024-11-24T02:46:17,871 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting edee406d41f0446b983a82baa823b402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416377380 2024-11-24T02:46:17,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416437877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416437878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416437878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416437878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416437887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:17,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:17,907 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#13 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:17,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741849_1025 (size=12001) 2024-11-24T02:46:17,909 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/ee3f57bc3f5d4b7cb64117f2737208ee is 50, key is test_row_0/C:col10/1732416377380/Put/seqid=0 2024-11-24T02:46:17,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/095da2814aec40eb879740514bba2f0e 2024-11-24T02:46:17,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/93640ef446d041a5bfbf5b39fcccf309 is 50, key is test_row_0/C:col10/1732416377500/Put/seqid=0 2024-11-24T02:46:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T02:46:17,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:17,958 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-24T02:46:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:46:17,962 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:17,964 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:17,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:17,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741850_1026 (size=12104) 2024-11-24T02:46:18,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741851_1027 (size=12001) 2024-11-24T02:46:18,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/93640ef446d041a5bfbf5b39fcccf309 2024-11-24T02:46:18,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/04c7743d0c2f4da18db106c4580d71cc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04c7743d0c2f4da18db106c4580d71cc 2024-11-24T02:46:18,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04c7743d0c2f4da18db106c4580d71cc, entries=200, sequenceid=75, filesize=14.0 K 2024-11-24T02:46:18,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/095da2814aec40eb879740514bba2f0e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/095da2814aec40eb879740514bba2f0e 2024-11-24T02:46:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:46:18,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/095da2814aec40eb879740514bba2f0e, entries=150, sequenceid=75, filesize=11.7 K 2024-11-24T02:46:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/93640ef446d041a5bfbf5b39fcccf309 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/93640ef446d041a5bfbf5b39fcccf309 2024-11-24T02:46:18,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416438087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416438097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416438098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416438098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/93640ef446d041a5bfbf5b39fcccf309, entries=150, sequenceid=75, filesize=11.7 K 2024-11-24T02:46:18,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 129703e293b1a78847241fc258ac8730 in 367ms, sequenceid=75, compaction requested=false 2024-11-24T02:46:18,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:18,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:18,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:18,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:46:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7ded0842617f4123bcbe6f44cfb82b46 is 50, key is test_row_0/A:col10/1732416377754/Put/seqid=0 2024-11-24T02:46:18,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741852_1028 (size=14341) 2024-11-24T02:46:18,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7ded0842617f4123bcbe6f44cfb82b46 2024-11-24T02:46:18,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b50b8905ae4e420992e11da284bdc88a is 50, key is test_row_0/B:col10/1732416377754/Put/seqid=0 2024-11-24T02:46:18,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741853_1029 (size=12001) 2024-11-24T02:46:18,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b50b8905ae4e420992e11da284bdc88a 2024-11-24T02:46:18,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/4a440f8120764bdeaee62aefcae741d3 is 50, key is test_row_0/C:col10/1732416377754/Put/seqid=0 2024-11-24T02:46:18,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:46:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:46:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741854_1030 (size=12001) 2024-11-24T02:46:18,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/4a440f8120764bdeaee62aefcae741d3 2024-11-24T02:46:18,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7ded0842617f4123bcbe6f44cfb82b46 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7ded0842617f4123bcbe6f44cfb82b46 2024-11-24T02:46:18,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7ded0842617f4123bcbe6f44cfb82b46, entries=200, sequenceid=88, filesize=14.0 K 2024-11-24T02:46:18,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416438344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b50b8905ae4e420992e11da284bdc88a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b50b8905ae4e420992e11da284bdc88a 2024-11-24T02:46:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b50b8905ae4e420992e11da284bdc88a, entries=150, sequenceid=88, filesize=11.7 K 2024-11-24T02:46:18,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/4a440f8120764bdeaee62aefcae741d3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/4a440f8120764bdeaee62aefcae741d3 2024-11-24T02:46:18,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/4a440f8120764bdeaee62aefcae741d3, entries=150, sequenceid=88, filesize=11.7 K 2024-11-24T02:46:18,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 129703e293b1a78847241fc258ac8730 in 285ms, sequenceid=88, compaction requested=true 2024-11-24T02:46:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:18,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:18,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:46:18,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:18,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T02:46:18,404 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 3 compacting, 2 eligible, 16 blocking 2024-11-24T02:46:18,407 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:46:18,407 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:46:18,408 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. because compaction request was cancelled 2024-11-24T02:46:18,408 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:18,408 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:18,413 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/ee3f57bc3f5d4b7cb64117f2737208ee as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ee3f57bc3f5d4b7cb64117f2737208ee 2024-11-24T02:46:18,414 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:18,414 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:18,414 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,415 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/afa85aa9fe5d45e5b26014ef943c2055, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04c7743d0c2f4da18db106c4580d71cc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7ded0842617f4123bcbe6f44cfb82b46] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=39.8 K 2024-11-24T02:46:18,415 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:18,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:46:18,416 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting afa85aa9fe5d45e5b26014ef943c2055, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416377380 2024-11-24T02:46:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:18,418 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 04c7743d0c2f4da18db106c4580d71cc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732416377500 2024-11-24T02:46:18,420 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ded0842617f4123bcbe6f44cfb82b46, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416377753 2024-11-24T02:46:18,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4d013612cddf44ab8982b0bafffcf6f0 is 50, key is test_row_0/A:col10/1732416378341/Put/seqid=0 2024-11-24T02:46:18,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:46:18,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:18,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,440 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into ee3f57bc3f5d4b7cb64117f2737208ee(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:18,443 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:18,443 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=13, startTime=1732416377681; duration=0sec 2024-11-24T02:46:18,443 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:18,443 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:18,444 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:18,451 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:18,451 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:18,452 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,452 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7a43a87368c844928aea94b74782bb57, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/095da2814aec40eb879740514bba2f0e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b50b8905ae4e420992e11da284bdc88a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=35.3 K 2024-11-24T02:46:18,454 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a43a87368c844928aea94b74782bb57, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416377380 2024-11-24T02:46:18,455 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095da2814aec40eb879740514bba2f0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732416377500 2024-11-24T02:46:18,460 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:18,461 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/33bdb74ec4d643bfa67653de0704b708 is 50, key is test_row_0/A:col10/1732416377754/Put/seqid=0 2024-11-24T02:46:18,462 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b50b8905ae4e420992e11da284bdc88a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416377754 2024-11-24T02:46:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416438449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416438453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416438459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416438466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416438467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741855_1031 (size=16681) 2024-11-24T02:46:18,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4d013612cddf44ab8982b0bafffcf6f0 2024-11-24T02:46:18,509 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:18,510 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/13bf2993fd124f7d9a43e00a4d1418ba is 50, key is test_row_0/B:col10/1732416377754/Put/seqid=0 2024-11-24T02:46:18,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/bbff7b362f6244249109a5cd09a892ee is 50, key is test_row_0/B:col10/1732416378341/Put/seqid=0 2024-11-24T02:46:18,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741856_1032 (size=12207) 2024-11-24T02:46:18,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741857_1033 (size=12207) 2024-11-24T02:46:18,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741858_1034 (size=12001) 2024-11-24T02:46:18,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416438577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416438579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/bbff7b362f6244249109a5cd09a892ee 2024-11-24T02:46:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:46:18,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:46:18,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:18,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416438588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416438590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,598 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/13bf2993fd124f7d9a43e00a4d1418ba as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/13bf2993fd124f7d9a43e00a4d1418ba 2024-11-24T02:46:18,598 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T02:46:18,623 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 13bf2993fd124f7d9a43e00a4d1418ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:18,623 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:18,623 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416378402; duration=0sec 2024-11-24T02:46:18,623 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:18,624 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:18,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/d57fc132d61045289af613f72967d18e is 50, key is test_row_0/C:col10/1732416378341/Put/seqid=0 2024-11-24T02:46:18,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741859_1035 (size=12001) 2024-11-24T02:46:18,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/d57fc132d61045289af613f72967d18e 2024-11-24T02:46:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416438677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4d013612cddf44ab8982b0bafffcf6f0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d013612cddf44ab8982b0bafffcf6f0 2024-11-24T02:46:18,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d013612cddf44ab8982b0bafffcf6f0, entries=250, sequenceid=115, filesize=16.3 K 2024-11-24T02:46:18,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/bbff7b362f6244249109a5cd09a892ee as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/bbff7b362f6244249109a5cd09a892ee 2024-11-24T02:46:18,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/bbff7b362f6244249109a5cd09a892ee, entries=150, sequenceid=115, filesize=11.7 K 2024-11-24T02:46:18,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/d57fc132d61045289af613f72967d18e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d57fc132d61045289af613f72967d18e 2024-11-24T02:46:18,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:46:18,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:18,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d57fc132d61045289af613f72967d18e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-24T02:46:18,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 129703e293b1a78847241fc258ac8730 in 344ms, sequenceid=115, compaction requested=true 2024-11-24T02:46:18,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:18,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:18,760 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-24T02:46:18,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:18,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:46:18,762 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:46:18,762 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:46:18,762 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. because compaction request was cancelled 2024-11-24T02:46:18,762 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:18,762 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:18,765 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:18,765 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:18,766 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,766 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ee3f57bc3f5d4b7cb64117f2737208ee, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/93640ef446d041a5bfbf5b39fcccf309, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/4a440f8120764bdeaee62aefcae741d3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d57fc132d61045289af613f72967d18e] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=47.0 K 2024-11-24T02:46:18,767 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee3f57bc3f5d4b7cb64117f2737208ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732416377380 2024-11-24T02:46:18,768 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93640ef446d041a5bfbf5b39fcccf309, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732416377500 2024-11-24T02:46:18,769 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a440f8120764bdeaee62aefcae741d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416377754 2024-11-24T02:46:18,769 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d57fc132d61045289af613f72967d18e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732416378341 2024-11-24T02:46:18,797 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:18,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:18,818 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#23 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:18,820 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/a3a4c08dd4b343b7852939166afd0ef8 is 50, key is test_row_0/C:col10/1732416378341/Put/seqid=0 2024-11-24T02:46:18,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/50ada166cb8a4ed6839dac38dc5c1abf is 50, key is test_row_0/A:col10/1732416378460/Put/seqid=0 2024-11-24T02:46:18,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741860_1036 (size=12241) 2024-11-24T02:46:18,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741861_1037 (size=12001) 2024-11-24T02:46:18,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/50ada166cb8a4ed6839dac38dc5c1abf 2024-11-24T02:46:18,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/7ec5b6f116d5497bb382fcaabf34ae54 is 50, key is test_row_0/B:col10/1732416378460/Put/seqid=0 2024-11-24T02:46:18,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741862_1038 (size=12001) 2024-11-24T02:46:18,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/7ec5b6f116d5497bb382fcaabf34ae54 2024-11-24T02:46:18,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:46:18,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:18,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:18,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:18,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/7fe9b4908cb24026a116dadac2bdabfb is 50, key is test_row_0/C:col10/1732416378460/Put/seqid=0 2024-11-24T02:46:18,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416438941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416438941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416438945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416438945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:18,963 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/33bdb74ec4d643bfa67653de0704b708 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/33bdb74ec4d643bfa67653de0704b708 2024-11-24T02:46:18,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741863_1039 (size=12001) 2024-11-24T02:46:18,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/7fe9b4908cb24026a116dadac2bdabfb 2024-11-24T02:46:18,983 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into 33bdb74ec4d643bfa67653de0704b708(size=11.9 K), total size for store is 28.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:18,984 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:18,984 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416378402; duration=0sec 2024-11-24T02:46:18,984 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:18,984 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:18,984 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T02:46:18,985 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:46:18,986 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:46:18,986 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. because compaction request was cancelled 2024-11-24T02:46:18,986 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:18,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/50ada166cb8a4ed6839dac38dc5c1abf as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/50ada166cb8a4ed6839dac38dc5c1abf 2024-11-24T02:46:18,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416438987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/50ada166cb8a4ed6839dac38dc5c1abf, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T02:46:19,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/7ec5b6f116d5497bb382fcaabf34ae54 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7ec5b6f116d5497bb382fcaabf34ae54 2024-11-24T02:46:19,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7ec5b6f116d5497bb382fcaabf34ae54, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T02:46:19,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/7fe9b4908cb24026a116dadac2bdabfb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/7fe9b4908cb24026a116dadac2bdabfb 2024-11-24T02:46:19,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/7fe9b4908cb24026a116dadac2bdabfb, entries=150, sequenceid=128, filesize=11.7 K 2024-11-24T02:46:19,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 129703e293b1a78847241fc258ac8730 in 235ms, sequenceid=128, compaction requested=true 2024-11-24T02:46:19,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:19,033 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:19,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:19,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:19,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:19,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:19,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:19,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:46:19,043 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:19,043 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:19,043 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:19,044 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/33bdb74ec4d643bfa67653de0704b708, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d013612cddf44ab8982b0bafffcf6f0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/50ada166cb8a4ed6839dac38dc5c1abf] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=39.9 K 2024-11-24T02:46:19,045 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 33bdb74ec4d643bfa67653de0704b708, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416377754 2024-11-24T02:46:19,046 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d013612cddf44ab8982b0bafffcf6f0, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732416378289 2024-11-24T02:46:19,047 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ada166cb8a4ed6839dac38dc5c1abf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416378440 2024-11-24T02:46:19,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T02:46:19,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:19,064 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:46:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:19,076 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:19,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cc7c5f37b5d042f89d36318efac45adf is 50, key is test_row_0/A:col10/1732416378941/Put/seqid=0 2024-11-24T02:46:19,077 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3cb00dba2d5e49b7beb315be4fab0042 is 50, key is test_row_0/A:col10/1732416378460/Put/seqid=0 2024-11-24T02:46:19,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416439087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416439089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:46:19,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741864_1040 (size=12151) 2024-11-24T02:46:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416439092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,114 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cc7c5f37b5d042f89d36318efac45adf 2024-11-24T02:46:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416439099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/400d762a95594222a36118b809eb13b6 is 50, key is test_row_0/B:col10/1732416378941/Put/seqid=0 2024-11-24T02:46:19,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741865_1041 (size=12309) 2024-11-24T02:46:19,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741866_1042 (size=12151) 2024-11-24T02:46:19,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416439207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416439207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416439217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416439228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,257 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/a3a4c08dd4b343b7852939166afd0ef8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a3a4c08dd4b343b7852939166afd0ef8 2024-11-24T02:46:19,276 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into a3a4c08dd4b343b7852939166afd0ef8(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:19,276 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:19,276 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=12, startTime=1732416378761; duration=0sec 2024-11-24T02:46:19,277 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:46:19,277 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:19,277 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:19,277 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:19,284 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:19,284 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:19,284 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:19,285 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/13bf2993fd124f7d9a43e00a4d1418ba, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/bbff7b362f6244249109a5cd09a892ee, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7ec5b6f116d5497bb382fcaabf34ae54] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=35.4 K 2024-11-24T02:46:19,286 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13bf2993fd124f7d9a43e00a4d1418ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416377754 2024-11-24T02:46:19,288 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbff7b362f6244249109a5cd09a892ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732416378341 2024-11-24T02:46:19,289 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec5b6f116d5497bb382fcaabf34ae54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416378440 2024-11-24T02:46:19,334 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:19,335 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/6b75bfdb7b164c1d91a7508b6189f36a is 50, key is test_row_0/B:col10/1732416378460/Put/seqid=0 2024-11-24T02:46:19,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741867_1043 (size=12309) 2024-11-24T02:46:19,385 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/6b75bfdb7b164c1d91a7508b6189f36a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6b75bfdb7b164c1d91a7508b6189f36a 2024-11-24T02:46:19,401 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 6b75bfdb7b164c1d91a7508b6189f36a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:19,401 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:19,401 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416379035; duration=0sec 2024-11-24T02:46:19,402 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:19,402 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:19,402 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T02:46:19,404 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T02:46:19,404 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T02:46:19,404 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. because compaction request was cancelled 2024-11-24T02:46:19,404 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:19,405 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:46:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416439417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416439427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416439427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416439447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416439497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,547 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T02:46:19,548 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T02:46:19,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T02:46:19,550 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T02:46:19,550 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T02:46:19,550 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T02:46:19,552 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T02:46:19,552 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T02:46:19,561 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3cb00dba2d5e49b7beb315be4fab0042 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3cb00dba2d5e49b7beb315be4fab0042 2024-11-24T02:46:19,567 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/400d762a95594222a36118b809eb13b6 2024-11-24T02:46:19,582 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into 3cb00dba2d5e49b7beb315be4fab0042(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:19,582 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:19,582 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416379033; duration=0sec 2024-11-24T02:46:19,583 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:19,583 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/6fa8ad6be7674cd8b5366f9e5cb0abfb is 50, key is test_row_0/C:col10/1732416378941/Put/seqid=0 2024-11-24T02:46:19,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741868_1044 (size=12151) 2024-11-24T02:46:19,602 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/6fa8ad6be7674cd8b5366f9e5cb0abfb 2024-11-24T02:46:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cc7c5f37b5d042f89d36318efac45adf as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cc7c5f37b5d042f89d36318efac45adf 2024-11-24T02:46:19,643 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cc7c5f37b5d042f89d36318efac45adf, entries=150, sequenceid=154, filesize=11.9 K 2024-11-24T02:46:19,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/400d762a95594222a36118b809eb13b6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/400d762a95594222a36118b809eb13b6 2024-11-24T02:46:19,662 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/400d762a95594222a36118b809eb13b6, entries=150, sequenceid=154, filesize=11.9 K 2024-11-24T02:46:19,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/6fa8ad6be7674cd8b5366f9e5cb0abfb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6fa8ad6be7674cd8b5366f9e5cb0abfb 2024-11-24T02:46:19,689 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6fa8ad6be7674cd8b5366f9e5cb0abfb, entries=150, sequenceid=154, filesize=11.9 K 2024-11-24T02:46:19,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 129703e293b1a78847241fc258ac8730 in 628ms, sequenceid=154, compaction requested=true 2024-11-24T02:46:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-24T02:46:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-24T02:46:19,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T02:46:19,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7320 sec 2024-11-24T02:46:19,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7430 sec 2024-11-24T02:46:19,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:19,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:46:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:19,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/2af94bba5ca64f2bb530cfd3a9d98ff8 is 50, key is test_row_0/A:col10/1732416379743/Put/seqid=0 2024-11-24T02:46:19,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741869_1045 (size=16931) 2024-11-24T02:46:19,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/2af94bba5ca64f2bb530cfd3a9d98ff8 2024-11-24T02:46:19,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416439818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416439832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/adc1b1c976c74cd6a56d9d640c4bebe9 is 50, key is test_row_0/B:col10/1732416379743/Put/seqid=0 2024-11-24T02:46:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416439836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416439850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741870_1046 (size=12151) 2024-11-24T02:46:19,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/adc1b1c976c74cd6a56d9d640c4bebe9 2024-11-24T02:46:19,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/3d9b852d80c144bba4b737681dc75e1e is 50, key is test_row_0/C:col10/1732416379743/Put/seqid=0 2024-11-24T02:46:19,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416439947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416439947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416439968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416439968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:19,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741871_1047 (size=12151) 2024-11-24T02:46:19,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/3d9b852d80c144bba4b737681dc75e1e 2024-11-24T02:46:20,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/2af94bba5ca64f2bb530cfd3a9d98ff8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/2af94bba5ca64f2bb530cfd3a9d98ff8 2024-11-24T02:46:20,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/2af94bba5ca64f2bb530cfd3a9d98ff8, entries=250, sequenceid=171, filesize=16.5 K 2024-11-24T02:46:20,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/adc1b1c976c74cd6a56d9d640c4bebe9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/adc1b1c976c74cd6a56d9d640c4bebe9 2024-11-24T02:46:20,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/adc1b1c976c74cd6a56d9d640c4bebe9, entries=150, sequenceid=171, filesize=11.9 K 2024-11-24T02:46:20,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/3d9b852d80c144bba4b737681dc75e1e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d9b852d80c144bba4b737681dc75e1e 2024-11-24T02:46:20,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d9b852d80c144bba4b737681dc75e1e, entries=150, sequenceid=171, filesize=11.9 K 2024-11-24T02:46:20,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 129703e293b1a78847241fc258ac8730 in 325ms, sequenceid=171, compaction requested=true 2024-11-24T02:46:20,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:20,069 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:20,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:20,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:20,069 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:20,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:20,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:20,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:20,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:20,071 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:20,071 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:20,071 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,071 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3cb00dba2d5e49b7beb315be4fab0042, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cc7c5f37b5d042f89d36318efac45adf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/2af94bba5ca64f2bb530cfd3a9d98ff8] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=40.4 K 2024-11-24T02:46:20,088 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:20,088 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:20,088 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,089 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6b75bfdb7b164c1d91a7508b6189f36a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/400d762a95594222a36118b809eb13b6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/adc1b1c976c74cd6a56d9d640c4bebe9] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=35.8 K 2024-11-24T02:46:20,090 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b75bfdb7b164c1d91a7508b6189f36a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416378440 2024-11-24T02:46:20,091 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 400d762a95594222a36118b809eb13b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416378919 2024-11-24T02:46:20,091 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cb00dba2d5e49b7beb315be4fab0042, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416378440 2024-11-24T02:46:20,092 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7c5f37b5d042f89d36318efac45adf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416378919 2024-11-24T02:46:20,092 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting adc1b1c976c74cd6a56d9d640c4bebe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416379089 2024-11-24T02:46:20,097 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2af94bba5ca64f2bb530cfd3a9d98ff8, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416379080 2024-11-24T02:46:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T02:46:20,108 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:20,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-24T02:46:20,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:46:20,114 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:20,116 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:20,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:20,137 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:20,138 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/bba3b5658b57414c92421ba648c51814 is 50, key is test_row_0/A:col10/1732416379743/Put/seqid=0 2024-11-24T02:46:20,146 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:20,147 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b7d07ae02bb74e73880f9af5396af18b is 50, key is test_row_0/B:col10/1732416379743/Put/seqid=0 2024-11-24T02:46:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:20,169 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:20,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:46:20,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741872_1048 (size=12561) 2024-11-24T02:46:20,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4661c816f1254a8eab47177d8afad0f4 is 50, key is test_row_0/A:col10/1732416380159/Put/seqid=0 2024-11-24T02:46:20,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741873_1049 (size=12561) 2024-11-24T02:46:20,195 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/bba3b5658b57414c92421ba648c51814 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/bba3b5658b57414c92421ba648c51814 2024-11-24T02:46:20,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741874_1050 (size=14541) 2024-11-24T02:46:20,216 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into bba3b5658b57414c92421ba648c51814(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:20,217 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:20,217 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416380068; duration=0sec 2024-11-24T02:46:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:46:20,217 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:20,217 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:20,217 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:20,219 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:20,219 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:20,219 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,220 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a3a4c08dd4b343b7852939166afd0ef8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/7fe9b4908cb24026a116dadac2bdabfb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6fa8ad6be7674cd8b5366f9e5cb0abfb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d9b852d80c144bba4b737681dc75e1e] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=47.4 K 2024-11-24T02:46:20,222 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3a4c08dd4b343b7852939166afd0ef8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732416378341 2024-11-24T02:46:20,223 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe9b4908cb24026a116dadac2bdabfb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732416378440 2024-11-24T02:46:20,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4661c816f1254a8eab47177d8afad0f4 2024-11-24T02:46:20,226 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fa8ad6be7674cd8b5366f9e5cb0abfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416378919 2024-11-24T02:46:20,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416440217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416440221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,235 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9b852d80c144bba4b737681dc75e1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416379089 2024-11-24T02:46:20,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416440227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416440231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/912ed22665d0492ea05cc90e90806486 is 50, key is test_row_0/B:col10/1732416380159/Put/seqid=0 2024-11-24T02:46:20,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:20,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,312 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:20,313 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/2e93986d3ac44cccb3a11a5167e107bd is 50, key is test_row_0/C:col10/1732416379743/Put/seqid=0 2024-11-24T02:46:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741875_1051 (size=12151) 2024-11-24T02:46:20,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416440347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416440352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416440348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416440357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741876_1052 (size=12527) 2024-11-24T02:46:20,389 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/2e93986d3ac44cccb3a11a5167e107bd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/2e93986d3ac44cccb3a11a5167e107bd 2024-11-24T02:46:20,404 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into 2e93986d3ac44cccb3a11a5167e107bd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:20,404 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:20,405 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=12, startTime=1732416380069; duration=0sec 2024-11-24T02:46:20,405 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:20,405 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:46:20,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:20,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416440519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,535 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416440567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416440567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416440567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416440577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,617 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b7d07ae02bb74e73880f9af5396af18b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7d07ae02bb74e73880f9af5396af18b 2024-11-24T02:46:20,641 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into b7d07ae02bb74e73880f9af5396af18b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:20,641 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:20,641 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416380069; duration=0sec 2024-11-24T02:46:20,642 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:20,642 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:20,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:46:20,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/912ed22665d0492ea05cc90e90806486 2024-11-24T02:46:20,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:20,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/011b9eaa128845aabde85f4de7bc0e53 is 50, key is test_row_0/C:col10/1732416380159/Put/seqid=0 2024-11-24T02:46:20,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741877_1053 (size=12151) 2024-11-24T02:46:20,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416440887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416440888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416440887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416440888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:20,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:20,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:20,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:20,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:21,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:21,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:21,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/011b9eaa128845aabde85f4de7bc0e53 2024-11-24T02:46:21,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:21,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:21,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:21,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4661c816f1254a8eab47177d8afad0f4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4661c816f1254a8eab47177d8afad0f4 2024-11-24T02:46:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:46:21,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4661c816f1254a8eab47177d8afad0f4, entries=200, sequenceid=195, filesize=14.2 K 2024-11-24T02:46:21,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/912ed22665d0492ea05cc90e90806486 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/912ed22665d0492ea05cc90e90806486 2024-11-24T02:46:21,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/912ed22665d0492ea05cc90e90806486, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T02:46:21,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/011b9eaa128845aabde85f4de7bc0e53 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/011b9eaa128845aabde85f4de7bc0e53 2024-11-24T02:46:21,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/011b9eaa128845aabde85f4de7bc0e53, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T02:46:21,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 129703e293b1a78847241fc258ac8730 in 1109ms, sequenceid=195, compaction requested=false 2024-11-24T02:46:21,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:21,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T02:46:21,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:21,387 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:46:21,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d193c17a78b04c41bab4cd27fded561e is 50, key is test_row_0/A:col10/1732416380220/Put/seqid=0 2024-11-24T02:46:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:21,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741878_1054 (size=12151) 2024-11-24T02:46:21,455 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d193c17a78b04c41bab4cd27fded561e 2024-11-24T02:46:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416441466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416441468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416441469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/53043e1d395b4915829405467ad9bbd5 is 50, key is test_row_0/B:col10/1732416380220/Put/seqid=0 2024-11-24T02:46:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416441475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741879_1055 (size=12151) 2024-11-24T02:46:21,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416441587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416441587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416441587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416441587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416441797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,801 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416441808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416441808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416441808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:21,915 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/53043e1d395b4915829405467ad9bbd5 2024-11-24T02:46:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/a6308b2e497c4d6ab3c04c4f20a81644 is 50, key is test_row_0/C:col10/1732416380220/Put/seqid=0 2024-11-24T02:46:21,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741880_1056 (size=12151) 2024-11-24T02:46:21,968 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/a6308b2e497c4d6ab3c04c4f20a81644 2024-11-24T02:46:21,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d193c17a78b04c41bab4cd27fded561e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d193c17a78b04c41bab4cd27fded561e 2024-11-24T02:46:22,012 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d193c17a78b04c41bab4cd27fded561e, entries=150, sequenceid=210, filesize=11.9 K 2024-11-24T02:46:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/53043e1d395b4915829405467ad9bbd5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/53043e1d395b4915829405467ad9bbd5 2024-11-24T02:46:22,029 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/53043e1d395b4915829405467ad9bbd5, entries=150, sequenceid=210, filesize=11.9 K 2024-11-24T02:46:22,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/a6308b2e497c4d6ab3c04c4f20a81644 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a6308b2e497c4d6ab3c04c4f20a81644 2024-11-24T02:46:22,047 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a6308b2e497c4d6ab3c04c4f20a81644, entries=150, sequenceid=210, filesize=11.9 K 2024-11-24T02:46:22,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 129703e293b1a78847241fc258ac8730 in 663ms, sequenceid=210, compaction requested=true 2024-11-24T02:46:22,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:22,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:22,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-24T02:46:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-24T02:46:22,058 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T02:46:22,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9370 sec 2024-11-24T02:46:22,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9500 sec 2024-11-24T02:46:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:22,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:46:22,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3d83c1cb071c41bbacf2de10282e176e is 50, key is test_row_0/A:col10/1732416381461/Put/seqid=0 2024-11-24T02:46:22,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741881_1057 (size=12151) 2024-11-24T02:46:22,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3d83c1cb071c41bbacf2de10282e176e 2024-11-24T02:46:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416442138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416442139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416442145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416442146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/92da17de6b224ceba9d102a8299d64fd is 50, key is test_row_0/B:col10/1732416381461/Put/seqid=0 2024-11-24T02:46:22,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741882_1058 (size=12151) 2024-11-24T02:46:22,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/92da17de6b224ceba9d102a8299d64fd 2024-11-24T02:46:22,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/cee492f32bcc4400b0b06eb84a659e81 is 50, key is test_row_0/C:col10/1732416381461/Put/seqid=0 2024-11-24T02:46:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T02:46:22,257 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:22,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-24T02:46:22,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,265 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416442257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:46:22,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416442262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416442262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416442262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,267 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:22,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741883_1059 (size=12151) 2024-11-24T02:46:22,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/cee492f32bcc4400b0b06eb84a659e81 2024-11-24T02:46:22,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3d83c1cb071c41bbacf2de10282e176e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3d83c1cb071c41bbacf2de10282e176e 2024-11-24T02:46:22,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3d83c1cb071c41bbacf2de10282e176e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T02:46:22,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/92da17de6b224ceba9d102a8299d64fd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/92da17de6b224ceba9d102a8299d64fd 2024-11-24T02:46:22,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/92da17de6b224ceba9d102a8299d64fd, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T02:46:22,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/cee492f32bcc4400b0b06eb84a659e81 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cee492f32bcc4400b0b06eb84a659e81 2024-11-24T02:46:22,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cee492f32bcc4400b0b06eb84a659e81, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T02:46:22,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 129703e293b1a78847241fc258ac8730 in 232ms, sequenceid=235, compaction requested=true 2024-11-24T02:46:22,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:22,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:22,342 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:22,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:22,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:22,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:22,343 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:22,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:22,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:22,345 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:22,345 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:22,345 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:22,345 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/bba3b5658b57414c92421ba648c51814, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4661c816f1254a8eab47177d8afad0f4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d193c17a78b04c41bab4cd27fded561e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3d83c1cb071c41bbacf2de10282e176e] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=50.2 K 2024-11-24T02:46:22,346 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bba3b5658b57414c92421ba648c51814, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416379089 2024-11-24T02:46:22,346 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:22,347 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:22,347 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:22,347 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7d07ae02bb74e73880f9af5396af18b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/912ed22665d0492ea05cc90e90806486, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/53043e1d395b4915829405467ad9bbd5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/92da17de6b224ceba9d102a8299d64fd] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=47.9 K 2024-11-24T02:46:22,347 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4661c816f1254a8eab47177d8afad0f4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732416379796 2024-11-24T02:46:22,347 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d07ae02bb74e73880f9af5396af18b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416379089 2024-11-24T02:46:22,348 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d193c17a78b04c41bab4cd27fded561e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732416380185 2024-11-24T02:46:22,348 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 912ed22665d0492ea05cc90e90806486, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732416379827 2024-11-24T02:46:22,349 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d83c1cb071c41bbacf2de10282e176e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416381461 2024-11-24T02:46:22,349 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 53043e1d395b4915829405467ad9bbd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732416380185 2024-11-24T02:46:22,350 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 92da17de6b224ceba9d102a8299d64fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416381461 2024-11-24T02:46:22,367 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:22,368 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/f8ff46da541446ee8e2571c1589813d0 is 50, key is test_row_0/A:col10/1732416381461/Put/seqid=0 2024-11-24T02:46:22,372 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:22,373 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/7204bdb21efa426183f28d1407d5eab4 is 50, key is test_row_0/B:col10/1732416381461/Put/seqid=0 2024-11-24T02:46:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:46:22,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741884_1060 (size=12697) 2024-11-24T02:46:22,407 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/f8ff46da541446ee8e2571c1589813d0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f8ff46da541446ee8e2571c1589813d0 2024-11-24T02:46:22,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741885_1061 (size=12697) 2024-11-24T02:46:22,420 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into f8ff46da541446ee8e2571c1589813d0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:22,420 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:22,420 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=12, startTime=1732416382342; duration=0sec 2024-11-24T02:46:22,421 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:22,421 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:22,421 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:22,425 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:22,425 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:22,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T02:46:22,425 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:22,425 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/2e93986d3ac44cccb3a11a5167e107bd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/011b9eaa128845aabde85f4de7bc0e53, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a6308b2e497c4d6ab3c04c4f20a81644, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cee492f32bcc4400b0b06eb84a659e81] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=47.8 K 2024-11-24T02:46:22,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:22,426 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:22,426 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e93986d3ac44cccb3a11a5167e107bd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416379089 2024-11-24T02:46:22,427 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 011b9eaa128845aabde85f4de7bc0e53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732416379827 2024-11-24T02:46:22,428 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6308b2e497c4d6ab3c04c4f20a81644, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732416380185 2024-11-24T02:46:22,429 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cee492f32bcc4400b0b06eb84a659e81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416381461 2024-11-24T02:46:22,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/93ad432fffdb409cb7b45ba78a047318 is 50, key is test_row_0/A:col10/1732416382122/Put/seqid=0 2024-11-24T02:46:22,455 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:22,458 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/0efe20dc850148b7b02575456e6d054a is 50, key is test_row_0/C:col10/1732416381461/Put/seqid=0 2024-11-24T02:46:22,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:22,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741886_1062 (size=12151) 2024-11-24T02:46:22,488 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/93ad432fffdb409cb7b45ba78a047318 2024-11-24T02:46:22,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e0f87769ecf14902b70d3a7ee68ef38c is 50, key is test_row_0/B:col10/1732416382122/Put/seqid=0 2024-11-24T02:46:22,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741887_1063 (size=12663) 2024-11-24T02:46:22,537 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/0efe20dc850148b7b02575456e6d054a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/0efe20dc850148b7b02575456e6d054a 2024-11-24T02:46:22,554 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into 0efe20dc850148b7b02575456e6d054a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:22,554 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:22,555 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=12, startTime=1732416382343; duration=0sec 2024-11-24T02:46:22,555 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:22,555 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:22,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741888_1064 (size=12151) 2024-11-24T02:46:22,577 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e0f87769ecf14902b70d3a7ee68ef38c 2024-11-24T02:46:22,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416442568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416442568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416442571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:46:22,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416442574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416442576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/3d88f3ed26ef482887b62de1090e97c1 is 50, key is test_row_0/C:col10/1732416382122/Put/seqid=0 2024-11-24T02:46:22,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741889_1065 (size=12151) 2024-11-24T02:46:22,622 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/3d88f3ed26ef482887b62de1090e97c1 2024-11-24T02:46:22,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/93ad432fffdb409cb7b45ba78a047318 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93ad432fffdb409cb7b45ba78a047318 2024-11-24T02:46:22,646 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93ad432fffdb409cb7b45ba78a047318, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T02:46:22,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e0f87769ecf14902b70d3a7ee68ef38c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e0f87769ecf14902b70d3a7ee68ef38c 2024-11-24T02:46:22,658 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e0f87769ecf14902b70d3a7ee68ef38c, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T02:46:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/3d88f3ed26ef482887b62de1090e97c1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d88f3ed26ef482887b62de1090e97c1 2024-11-24T02:46:22,681 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d88f3ed26ef482887b62de1090e97c1, entries=150, sequenceid=247, filesize=11.9 K 2024-11-24T02:46:22,683 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 129703e293b1a78847241fc258ac8730 in 257ms, sequenceid=247, compaction requested=false 2024-11-24T02:46:22,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:22,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:22,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-24T02:46:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-24T02:46:22,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T02:46:22,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 417 msec 2024-11-24T02:46:22,694 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:22,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:46:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:22,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 434 msec 2024-11-24T02:46:22,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416442701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416442709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416442709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416442711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416442713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/319b6702d25743a5819addd8f0928ef9 is 50, key is test_row_0/A:col10/1732416382567/Put/seqid=0 2024-11-24T02:46:22,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741890_1066 (size=12301) 2024-11-24T02:46:22,816 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/7204bdb21efa426183f28d1407d5eab4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7204bdb21efa426183f28d1407d5eab4 2024-11-24T02:46:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416442817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416442817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416442827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416442827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416442827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:22,832 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 7204bdb21efa426183f28d1407d5eab4(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:22,832 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:22,832 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=12, startTime=1732416382342; duration=0sec 2024-11-24T02:46:22,833 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:22,833 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T02:46:22,897 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:22,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-24T02:46:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:46:22,901 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:22,902 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:22,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:46:23,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416443027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416443030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416443037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416443037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416443037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/319b6702d25743a5819addd8f0928ef9 2024-11-24T02:46:23,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/fcae20d619744950a5948d468930ffa7 is 50, key is test_row_0/B:col10/1732416382567/Put/seqid=0 2024-11-24T02:46:23,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:23,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:46:23,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741891_1067 (size=12301) 2024-11-24T02:46:23,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416443337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416443338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416443347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416443347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416443347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:23,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:23,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:46:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/fcae20d619744950a5948d468930ffa7 2024-11-24T02:46:23,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5ded04b6768f4100ad0157af344fe87c is 50, key is test_row_0/C:col10/1732416382567/Put/seqid=0 2024-11-24T02:46:23,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741892_1068 (size=12301) 2024-11-24T02:46:23,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416443847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416443848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416443857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416443867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416443867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:23,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:23,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:23,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:23,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:46:24,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5ded04b6768f4100ad0157af344fe87c 2024-11-24T02:46:24,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/319b6702d25743a5819addd8f0928ef9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/319b6702d25743a5819addd8f0928ef9 2024-11-24T02:46:24,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/319b6702d25743a5819addd8f0928ef9, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T02:46:24,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/fcae20d619744950a5948d468930ffa7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fcae20d619744950a5948d468930ffa7 2024-11-24T02:46:24,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fcae20d619744950a5948d468930ffa7, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T02:46:24,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5ded04b6768f4100ad0157af344fe87c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5ded04b6768f4100ad0157af344fe87c 2024-11-24T02:46:24,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5ded04b6768f4100ad0157af344fe87c, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T02:46:24,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 129703e293b1a78847241fc258ac8730 in 1439ms, sequenceid=275, compaction requested=true 2024-11-24T02:46:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:24,134 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:24,134 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:24,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:24,136 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:24,137 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:24,137 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:24,137 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7204bdb21efa426183f28d1407d5eab4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e0f87769ecf14902b70d3a7ee68ef38c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fcae20d619744950a5948d468930ffa7] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=36.3 K 2024-11-24T02:46:24,137 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:24,137 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:24,137 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:24,138 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f8ff46da541446ee8e2571c1589813d0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93ad432fffdb409cb7b45ba78a047318, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/319b6702d25743a5819addd8f0928ef9] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=36.3 K 2024-11-24T02:46:24,138 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8ff46da541446ee8e2571c1589813d0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416381461 2024-11-24T02:46:24,139 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 7204bdb21efa426183f28d1407d5eab4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416381461 2024-11-24T02:46:24,139 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93ad432fffdb409cb7b45ba78a047318, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732416382122 2024-11-24T02:46:24,139 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e0f87769ecf14902b70d3a7ee68ef38c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732416382122 2024-11-24T02:46:24,140 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting fcae20d619744950a5948d468930ffa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416382567 2024-11-24T02:46:24,141 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 319b6702d25743a5819addd8f0928ef9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416382567 2024-11-24T02:46:24,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T02:46:24,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:24,145 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-24T02:46:24,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/f3649f79765643e7bc7e77f1e2c7b71b is 50, key is test_row_0/A:col10/1732416382703/Put/seqid=0 2024-11-24T02:46:24,173 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:24,174 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/443a7519afee4db0bb8d0b3bb00286f3 is 50, key is test_row_0/B:col10/1732416382567/Put/seqid=0 2024-11-24T02:46:24,191 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:24,192 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7de55ad0a92043859051c11a123b686b is 50, key is test_row_0/A:col10/1732416382567/Put/seqid=0 2024-11-24T02:46:24,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741893_1069 (size=9857) 2024-11-24T02:46:24,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741894_1070 (size=12949) 2024-11-24T02:46:24,251 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/443a7519afee4db0bb8d0b3bb00286f3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/443a7519afee4db0bb8d0b3bb00286f3 2024-11-24T02:46:24,265 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 443a7519afee4db0bb8d0b3bb00286f3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:24,265 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:24,265 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416384134; duration=0sec 2024-11-24T02:46:24,266 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:24,266 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:24,266 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:24,268 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:24,268 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:24,268 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:24,269 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/0efe20dc850148b7b02575456e6d054a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d88f3ed26ef482887b62de1090e97c1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5ded04b6768f4100ad0157af344fe87c] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=36.2 K 2024-11-24T02:46:24,270 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0efe20dc850148b7b02575456e6d054a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416381461 2024-11-24T02:46:24,271 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d88f3ed26ef482887b62de1090e97c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732416382122 2024-11-24T02:46:24,271 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ded04b6768f4100ad0157af344fe87c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416382567 2024-11-24T02:46:24,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741895_1071 (size=12949) 2024-11-24T02:46:24,292 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:24,293 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/1d259267c0c94bed972543cdfffa60f7 is 50, key is test_row_0/C:col10/1732416382567/Put/seqid=0 2024-11-24T02:46:24,297 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/7de55ad0a92043859051c11a123b686b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7de55ad0a92043859051c11a123b686b 2024-11-24T02:46:24,315 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into 7de55ad0a92043859051c11a123b686b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:24,315 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:24,315 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416384134; duration=0sec 2024-11-24T02:46:24,315 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:24,315 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:24,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741896_1072 (size=12915) 2024-11-24T02:46:24,330 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/1d259267c0c94bed972543cdfffa60f7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/1d259267c0c94bed972543cdfffa60f7 2024-11-24T02:46:24,341 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into 1d259267c0c94bed972543cdfffa60f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:24,341 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:24,341 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=13, startTime=1732416384134; duration=0sec 2024-11-24T02:46:24,341 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:24,341 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:24,639 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/f3649f79765643e7bc7e77f1e2c7b71b 2024-11-24T02:46:24,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e6abc8892fab4ce8b7783bf2a079432b is 50, key is test_row_0/B:col10/1732416382703/Put/seqid=0 2024-11-24T02:46:24,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741897_1073 (size=9857) 2024-11-24T02:46:24,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:24,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416444917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:24,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416444929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416444932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:24,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416444932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:24,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416444937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:24,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416445036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:46:25,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416445047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416445047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416445047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416445047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e6abc8892fab4ce8b7783bf2a079432b 2024-11-24T02:46:25,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/41af04c46a7f4280b80367d4f35d425d is 50, key is test_row_0/C:col10/1732416382703/Put/seqid=0 2024-11-24T02:46:25,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741898_1074 (size=9857) 2024-11-24T02:46:25,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416445246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416445257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416445257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416445257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416445257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,495 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/41af04c46a7f4280b80367d4f35d425d 2024-11-24T02:46:25,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/f3649f79765643e7bc7e77f1e2c7b71b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f3649f79765643e7bc7e77f1e2c7b71b 2024-11-24T02:46:25,515 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f3649f79765643e7bc7e77f1e2c7b71b, entries=100, sequenceid=285, filesize=9.6 K 2024-11-24T02:46:25,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e6abc8892fab4ce8b7783bf2a079432b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e6abc8892fab4ce8b7783bf2a079432b 2024-11-24T02:46:25,527 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e6abc8892fab4ce8b7783bf2a079432b, entries=100, sequenceid=285, filesize=9.6 K 2024-11-24T02:46:25,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/41af04c46a7f4280b80367d4f35d425d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/41af04c46a7f4280b80367d4f35d425d 2024-11-24T02:46:25,542 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/41af04c46a7f4280b80367d4f35d425d, entries=100, sequenceid=285, filesize=9.6 K 2024-11-24T02:46:25,543 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 129703e293b1a78847241fc258ac8730 in 1398ms, sequenceid=285, compaction requested=false 2024-11-24T02:46:25,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:25,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:25,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-24T02:46:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-24T02:46:25,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T02:46:25,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6430 sec 2024-11-24T02:46:25,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.6510 sec 2024-11-24T02:46:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:25,562 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:25,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-24T02:46:25,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/1fd38f73dc454a3fa1b791dba84be68d is 50, key is test_row_0/A:col10/1732416384928/Put/seqid=0 2024-11-24T02:46:25,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741899_1075 (size=14741) 2024-11-24T02:46:25,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416445577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416445577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416445616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416445616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416445617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416445727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416445727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416445727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416445937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416445937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:25,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416445937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:25,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/1fd38f73dc454a3fa1b791dba84be68d 2024-11-24T02:46:26,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/c19f5cbafbe447319715a3c6268bcb06 is 50, key is test_row_0/B:col10/1732416384928/Put/seqid=0 2024-11-24T02:46:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741900_1076 (size=12301) 2024-11-24T02:46:26,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/c19f5cbafbe447319715a3c6268bcb06 2024-11-24T02:46:26,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/78f93771cb4a4f40b2fbf5f48aca4330 is 50, key is test_row_0/C:col10/1732416384928/Put/seqid=0 2024-11-24T02:46:26,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741901_1077 (size=12301) 2024-11-24T02:46:26,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416446127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416446127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416446247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416446247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416446247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/78f93771cb4a4f40b2fbf5f48aca4330 2024-11-24T02:46:26,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/1fd38f73dc454a3fa1b791dba84be68d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/1fd38f73dc454a3fa1b791dba84be68d 2024-11-24T02:46:26,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/1fd38f73dc454a3fa1b791dba84be68d, entries=200, sequenceid=317, filesize=14.4 K 2024-11-24T02:46:26,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/c19f5cbafbe447319715a3c6268bcb06 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/c19f5cbafbe447319715a3c6268bcb06 2024-11-24T02:46:26,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/c19f5cbafbe447319715a3c6268bcb06, entries=150, sequenceid=317, filesize=12.0 K 2024-11-24T02:46:26,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/78f93771cb4a4f40b2fbf5f48aca4330 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/78f93771cb4a4f40b2fbf5f48aca4330 2024-11-24T02:46:26,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/78f93771cb4a4f40b2fbf5f48aca4330, entries=150, sequenceid=317, filesize=12.0 K 2024-11-24T02:46:26,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 129703e293b1a78847241fc258ac8730 in 931ms, sequenceid=317, compaction requested=true 2024-11-24T02:46:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:26,494 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:26,494 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:26,495 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37547 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:26,496 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:26,496 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:26,496 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:26,496 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:26,496 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:26,496 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7de55ad0a92043859051c11a123b686b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f3649f79765643e7bc7e77f1e2c7b71b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/1fd38f73dc454a3fa1b791dba84be68d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=36.7 K 2024-11-24T02:46:26,496 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/443a7519afee4db0bb8d0b3bb00286f3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e6abc8892fab4ce8b7783bf2a079432b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/c19f5cbafbe447319715a3c6268bcb06] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=34.3 K 2024-11-24T02:46:26,497 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 443a7519afee4db0bb8d0b3bb00286f3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416382567 2024-11-24T02:46:26,497 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de55ad0a92043859051c11a123b686b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416382567 2024-11-24T02:46:26,498 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3649f79765643e7bc7e77f1e2c7b71b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732416382703 2024-11-24T02:46:26,498 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e6abc8892fab4ce8b7783bf2a079432b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732416382703 2024-11-24T02:46:26,498 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fd38f73dc454a3fa1b791dba84be68d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732416384919 2024-11-24T02:46:26,498 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c19f5cbafbe447319715a3c6268bcb06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732416384919 2024-11-24T02:46:26,512 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:26,513 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:26,513 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/796a6875916440e7a6be979086e16284 is 50, key is test_row_0/B:col10/1732416384928/Put/seqid=0 2024-11-24T02:46:26,513 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4d84f76b16c749e5bb1c5c3ec13ca29c is 50, key is test_row_0/A:col10/1732416384928/Put/seqid=0 2024-11-24T02:46:26,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741903_1079 (size=13051) 2024-11-24T02:46:26,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741902_1078 (size=13051) 2024-11-24T02:46:26,557 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/796a6875916440e7a6be979086e16284 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/796a6875916440e7a6be979086e16284 2024-11-24T02:46:26,570 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 796a6875916440e7a6be979086e16284(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:26,570 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:26,570 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416386494; duration=0sec 2024-11-24T02:46:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:26,573 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:26,573 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:26,573 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:26,573 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/1d259267c0c94bed972543cdfffa60f7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/41af04c46a7f4280b80367d4f35d425d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/78f93771cb4a4f40b2fbf5f48aca4330] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=34.3 K 2024-11-24T02:46:26,574 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d259267c0c94bed972543cdfffa60f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732416382567 2024-11-24T02:46:26,575 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 41af04c46a7f4280b80367d4f35d425d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732416382703 2024-11-24T02:46:26,575 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 78f93771cb4a4f40b2fbf5f48aca4330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732416384919 2024-11-24T02:46:26,592 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#67 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:26,593 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/67fd31aed34c458baab58e799c11c55a is 50, key is test_row_0/C:col10/1732416384928/Put/seqid=0 2024-11-24T02:46:26,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741904_1080 (size=13017) 2024-11-24T02:46:26,631 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/67fd31aed34c458baab58e799c11c55a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/67fd31aed34c458baab58e799c11c55a 2024-11-24T02:46:26,641 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into 67fd31aed34c458baab58e799c11c55a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:26,641 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:26,641 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=13, startTime=1732416386494; duration=0sec 2024-11-24T02:46:26,644 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:26,645 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:26,761 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:26,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:26,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/04b3a9b75a2b4f06a888614cdd68d570 is 50, key is test_row_0/A:col10/1732416386761/Put/seqid=0 2024-11-24T02:46:26,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741905_1081 (size=12297) 2024-11-24T02:46:26,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/04b3a9b75a2b4f06a888614cdd68d570 2024-11-24T02:46:26,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/1558b6938a494771a6482ac3469308b1 is 50, key is test_row_0/B:col10/1732416386761/Put/seqid=0 2024-11-24T02:46:26,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416446813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416446818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416446818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741906_1082 (size=9857) 2024-11-24T02:46:26,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/1558b6938a494771a6482ac3469308b1 2024-11-24T02:46:26,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/cc417039ec30464f9b2810728e11064e is 50, key is test_row_0/C:col10/1732416386761/Put/seqid=0 2024-11-24T02:46:26,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741907_1083 (size=9857) 2024-11-24T02:46:26,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416446927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416446927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,930 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416446927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:26,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:26,944 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/4d84f76b16c749e5bb1c5c3ec13ca29c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d84f76b16c749e5bb1c5c3ec13ca29c 2024-11-24T02:46:26,953 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into 4d84f76b16c749e5bb1c5c3ec13ca29c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:26,953 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:26,953 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416386494; duration=0sec 2024-11-24T02:46:26,954 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:26,954 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T02:46:27,067 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:27,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-24T02:46:27,070 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:27,071 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:27,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:27,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:46:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416447137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416447137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416447137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416447147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416447147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,151 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,153 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:46:27,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:46:27,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:27,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:27,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:27,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:27,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/cc417039ec30464f9b2810728e11064e 2024-11-24T02:46:27,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/04b3a9b75a2b4f06a888614cdd68d570 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04b3a9b75a2b4f06a888614cdd68d570 2024-11-24T02:46:27,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04b3a9b75a2b4f06a888614cdd68d570, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:46:27,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/1558b6938a494771a6482ac3469308b1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/1558b6938a494771a6482ac3469308b1 2024-11-24T02:46:27,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/1558b6938a494771a6482ac3469308b1, entries=100, sequenceid=331, filesize=9.6 K 2024-11-24T02:46:27,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/cc417039ec30464f9b2810728e11064e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cc417039ec30464f9b2810728e11064e 2024-11-24T02:46:27,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cc417039ec30464f9b2810728e11064e, entries=100, sequenceid=331, filesize=9.6 K 2024-11-24T02:46:27,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 129703e293b1a78847241fc258ac8730 in 551ms, sequenceid=331, compaction requested=false 2024-11-24T02:46:27,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:27,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T02:46:27,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:27,379 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:46:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:46:27,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cba15169e6a04161a77b4846ccc163e8 is 50, key is test_row_0/A:col10/1732416386805/Put/seqid=0 2024-11-24T02:46:27,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741908_1084 (size=12301) 2024-11-24T02:46:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:27,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416447462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416447465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416447473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416447576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416447576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416447587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:46:27,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416447786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416447787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:27,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416447797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:27,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:27,807 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cba15169e6a04161a77b4846ccc163e8 2024-11-24T02:46:27,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/937358dcb3014360bb75c24e66d54c7e is 50, key is test_row_0/B:col10/1732416386805/Put/seqid=0 2024-11-24T02:46:27,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741909_1085 (size=12301) 2024-11-24T02:46:28,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:28,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416448096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416448096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:28,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416448107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:46:28,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/937358dcb3014360bb75c24e66d54c7e 2024-11-24T02:46:28,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/64e06251ff004e448d0c2ff9ae76bc87 is 50, key is test_row_0/C:col10/1732416386805/Put/seqid=0 2024-11-24T02:46:28,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741910_1086 (size=12301) 2024-11-24T02:46:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:28,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416448607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416448607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:28,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416448616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:28,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:28,643 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/64e06251ff004e448d0c2ff9ae76bc87 2024-11-24T02:46:28,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cba15169e6a04161a77b4846ccc163e8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cba15169e6a04161a77b4846ccc163e8 2024-11-24T02:46:28,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cba15169e6a04161a77b4846ccc163e8, entries=150, sequenceid=356, filesize=12.0 K 2024-11-24T02:46:28,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/937358dcb3014360bb75c24e66d54c7e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/937358dcb3014360bb75c24e66d54c7e 2024-11-24T02:46:28,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/937358dcb3014360bb75c24e66d54c7e, entries=150, sequenceid=356, filesize=12.0 K 2024-11-24T02:46:28,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/64e06251ff004e448d0c2ff9ae76bc87 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/64e06251ff004e448d0c2ff9ae76bc87 2024-11-24T02:46:28,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/64e06251ff004e448d0c2ff9ae76bc87, entries=150, sequenceid=356, filesize=12.0 K 2024-11-24T02:46:28,671 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 129703e293b1a78847241fc258ac8730 in 1292ms, sequenceid=356, compaction requested=true 2024-11-24T02:46:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:28,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:28,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-24T02:46:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-24T02:46:28,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T02:46:28,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6020 sec 2024-11-24T02:46:28,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6090 sec 2024-11-24T02:46:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:29,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:29,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:46:29,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/ad33d93315ba4d8c89563826c2e8e642 is 50, key is test_row_0/A:col10/1732416387461/Put/seqid=0 2024-11-24T02:46:29,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741911_1087 (size=14741) 2024-11-24T02:46:29,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416449207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416449208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T02:46:29,227 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:29,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-24T02:46:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:46:29,229 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:29,230 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:29,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:29,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416449316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416449316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:46:29,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:46:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416449526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416449526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:46:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:46:29,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/ad33d93315ba4d8c89563826c2e8e642 2024-11-24T02:46:29,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e7ad8971a48443af838ab3680025e49a is 50, key is test_row_0/B:col10/1732416387461/Put/seqid=0 2024-11-24T02:46:29,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741912_1088 (size=12301) 2024-11-24T02:46:29,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e7ad8971a48443af838ab3680025e49a 2024-11-24T02:46:29,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/11304f31808d47309c487b2b36c0bceb is 50, key is test_row_0/C:col10/1732416387461/Put/seqid=0 2024-11-24T02:46:29,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416449617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,619 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741913_1089 (size=12301) 2024-11-24T02:46:29,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416449627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416449627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,629 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,631 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:46:29,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:29,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416449837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416449837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:29,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:46:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:46:29,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:46:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:29,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:30,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/11304f31808d47309c487b2b36c0bceb 2024-11-24T02:46:30,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/ad33d93315ba4d8c89563826c2e8e642 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/ad33d93315ba4d8c89563826c2e8e642 2024-11-24T02:46:30,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/ad33d93315ba4d8c89563826c2e8e642, entries=200, sequenceid=369, filesize=14.4 K 2024-11-24T02:46:30,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e7ad8971a48443af838ab3680025e49a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e7ad8971a48443af838ab3680025e49a 2024-11-24T02:46:30,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e7ad8971a48443af838ab3680025e49a, entries=150, sequenceid=369, filesize=12.0 K 2024-11-24T02:46:30,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/11304f31808d47309c487b2b36c0bceb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/11304f31808d47309c487b2b36c0bceb 2024-11-24T02:46:30,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/11304f31808d47309c487b2b36c0bceb, entries=150, sequenceid=369, filesize=12.0 K 2024-11-24T02:46:30,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 129703e293b1a78847241fc258ac8730 in 878ms, sequenceid=369, compaction requested=true 2024-11-24T02:46:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:30,046 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:30,046 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:30,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:30,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:30,048 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52390 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:30,048 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:30,048 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:30,048 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:30,048 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:30,048 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:30,048 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/796a6875916440e7a6be979086e16284, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/1558b6938a494771a6482ac3469308b1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/937358dcb3014360bb75c24e66d54c7e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e7ad8971a48443af838ab3680025e49a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=46.4 K 2024-11-24T02:46:30,048 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d84f76b16c749e5bb1c5c3ec13ca29c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04b3a9b75a2b4f06a888614cdd68d570, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cba15169e6a04161a77b4846ccc163e8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/ad33d93315ba4d8c89563826c2e8e642] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=51.2 K 2024-11-24T02:46:30,049 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d84f76b16c749e5bb1c5c3ec13ca29c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732416384919 2024-11-24T02:46:30,049 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 796a6875916440e7a6be979086e16284, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732416384919 2024-11-24T02:46:30,049 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b3a9b75a2b4f06a888614cdd68d570, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416386758 2024-11-24T02:46:30,049 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1558b6938a494771a6482ac3469308b1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416386758 2024-11-24T02:46:30,049 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba15169e6a04161a77b4846ccc163e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732416386782 2024-11-24T02:46:30,049 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 937358dcb3014360bb75c24e66d54c7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732416386782 2024-11-24T02:46:30,050 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad33d93315ba4d8c89563826c2e8e642, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732416387449 2024-11-24T02:46:30,050 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ad8971a48443af838ab3680025e49a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732416387449 2024-11-24T02:46:30,060 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:30,060 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:30,060 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/a0d021ba2f3045fea19fde27b64c87d6 is 50, key is test_row_0/A:col10/1732416387461/Put/seqid=0 2024-11-24T02:46:30,061 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/67e00f657f0c428ab1e4d7761b465aa8 is 50, key is test_row_0/B:col10/1732416387461/Put/seqid=0 2024-11-24T02:46:30,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741914_1090 (size=13187) 2024-11-24T02:46:30,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741915_1091 (size=13187) 2024-11-24T02:46:30,082 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/67e00f657f0c428ab1e4d7761b465aa8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/67e00f657f0c428ab1e4d7761b465aa8 2024-11-24T02:46:30,090 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 67e00f657f0c428ab1e4d7761b465aa8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:30,090 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:30,090 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=12, startTime=1732416390046; duration=0sec 2024-11-24T02:46:30,090 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:30,090 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:30,090 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:30,091 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:30,091 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:30,091 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:30,092 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/67fd31aed34c458baab58e799c11c55a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cc417039ec30464f9b2810728e11064e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/64e06251ff004e448d0c2ff9ae76bc87, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/11304f31808d47309c487b2b36c0bceb] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=46.4 K 2024-11-24T02:46:30,092 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 67fd31aed34c458baab58e799c11c55a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732416384919 2024-11-24T02:46:30,092 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting cc417039ec30464f9b2810728e11064e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416386758 2024-11-24T02:46:30,093 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e06251ff004e448d0c2ff9ae76bc87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732416386782 2024-11-24T02:46:30,093 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 11304f31808d47309c487b2b36c0bceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732416387449 2024-11-24T02:46:30,103 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:30,103 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5cf055a92cfa42eead34205c056b8b01 is 50, key is test_row_0/C:col10/1732416387461/Put/seqid=0 2024-11-24T02:46:30,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741916_1092 (size=13153) 2024-11-24T02:46:30,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T02:46:30,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:30,149 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T02:46:30,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/da9ea05663764653854647038bba7c2b is 50, key is test_row_0/A:col10/1732416389207/Put/seqid=0 2024-11-24T02:46:30,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741917_1093 (size=12301) 2024-11-24T02:46:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:30,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416450367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:30,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416450371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:46:30,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416450476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:30,483 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/a0d021ba2f3045fea19fde27b64c87d6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a0d021ba2f3045fea19fde27b64c87d6 2024-11-24T02:46:30,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416450487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:30,490 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into a0d021ba2f3045fea19fde27b64c87d6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:30,490 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:30,491 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=12, startTime=1732416390046; duration=0sec 2024-11-24T02:46:30,491 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:30,491 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:30,514 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5cf055a92cfa42eead34205c056b8b01 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5cf055a92cfa42eead34205c056b8b01 2024-11-24T02:46:30,520 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into 5cf055a92cfa42eead34205c056b8b01(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:30,520 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:30,520 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=12, startTime=1732416390046; duration=0sec 2024-11-24T02:46:30,521 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:30,521 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:30,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/da9ea05663764653854647038bba7c2b 2024-11-24T02:46:30,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/0087702e7c6140189ce539f11458a39c is 50, key is test_row_0/B:col10/1732416389207/Put/seqid=0 2024-11-24T02:46:30,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741918_1094 (size=12301) 2024-11-24T02:46:30,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416450686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:30,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:30,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416450696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:30,991 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/0087702e7c6140189ce539f11458a39c 2024-11-24T02:46:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416450997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/03b4924fe18a44c2a0e2ec06dc44dca1 is 50, key is test_row_0/C:col10/1732416389207/Put/seqid=0 2024-11-24T02:46:31,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416451007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741919_1095 (size=12301) 2024-11-24T02:46:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:46:31,410 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/03b4924fe18a44c2a0e2ec06dc44dca1 2024-11-24T02:46:31,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/da9ea05663764653854647038bba7c2b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/da9ea05663764653854647038bba7c2b 2024-11-24T02:46:31,420 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/da9ea05663764653854647038bba7c2b, entries=150, sequenceid=393, filesize=12.0 K 2024-11-24T02:46:31,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/0087702e7c6140189ce539f11458a39c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0087702e7c6140189ce539f11458a39c 2024-11-24T02:46:31,426 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0087702e7c6140189ce539f11458a39c, entries=150, sequenceid=393, filesize=12.0 K 2024-11-24T02:46:31,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/03b4924fe18a44c2a0e2ec06dc44dca1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/03b4924fe18a44c2a0e2ec06dc44dca1 2024-11-24T02:46:31,433 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/03b4924fe18a44c2a0e2ec06dc44dca1, entries=150, sequenceid=393, filesize=12.0 K 2024-11-24T02:46:31,434 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 129703e293b1a78847241fc258ac8730 in 1285ms, sequenceid=393, compaction requested=false 2024-11-24T02:46:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-24T02:46:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-24T02:46:31,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T02:46:31,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-11-24T02:46:31,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2120 sec 2024-11-24T02:46:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:31,509 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:31,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:46:31,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d83386230c1f4d37805541dff9d4c00f is 50, key is test_row_0/A:col10/1732416390370/Put/seqid=0 2024-11-24T02:46:31,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741920_1096 (size=12301) 2024-11-24T02:46:31,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416451560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416451561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416451636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416451636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,638 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,638 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416451658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,660 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416451666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416451666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416451877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416451877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:31,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:31,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d83386230c1f4d37805541dff9d4c00f 2024-11-24T02:46:31,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e439cf7005f145dfbc07cee90598f1c7 is 50, key is test_row_0/B:col10/1732416390370/Put/seqid=0 2024-11-24T02:46:31,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741921_1097 (size=12301) 2024-11-24T02:46:32,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416452187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:32,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416452187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:32,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e439cf7005f145dfbc07cee90598f1c7 2024-11-24T02:46:32,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/40743b85eae241069a7900a1e3b9ddeb is 50, key is test_row_0/C:col10/1732416390370/Put/seqid=0 2024-11-24T02:46:32,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741922_1098 (size=12301) 2024-11-24T02:46:32,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416452697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:32,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416452700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:32,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/40743b85eae241069a7900a1e3b9ddeb 2024-11-24T02:46:32,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d83386230c1f4d37805541dff9d4c00f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d83386230c1f4d37805541dff9d4c00f 2024-11-24T02:46:32,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d83386230c1f4d37805541dff9d4c00f, entries=150, sequenceid=409, filesize=12.0 K 2024-11-24T02:46:32,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e439cf7005f145dfbc07cee90598f1c7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e439cf7005f145dfbc07cee90598f1c7 2024-11-24T02:46:32,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e439cf7005f145dfbc07cee90598f1c7, entries=150, sequenceid=409, filesize=12.0 K 2024-11-24T02:46:32,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/40743b85eae241069a7900a1e3b9ddeb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/40743b85eae241069a7900a1e3b9ddeb 2024-11-24T02:46:32,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/40743b85eae241069a7900a1e3b9ddeb, entries=150, sequenceid=409, filesize=12.0 K 2024-11-24T02:46:32,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 129703e293b1a78847241fc258ac8730 in 1286ms, sequenceid=409, compaction requested=true 2024-11-24T02:46:32,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:32,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:32,795 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:32,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:32,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:32,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:32,795 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:32,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:32,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:32,797 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:32,797 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:32,797 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:32,797 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:32,797 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:32,797 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:32,797 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/67e00f657f0c428ab1e4d7761b465aa8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0087702e7c6140189ce539f11458a39c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e439cf7005f145dfbc07cee90598f1c7] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=36.9 K 2024-11-24T02:46:32,797 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a0d021ba2f3045fea19fde27b64c87d6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/da9ea05663764653854647038bba7c2b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d83386230c1f4d37805541dff9d4c00f] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=36.9 K 2024-11-24T02:46:32,798 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e00f657f0c428ab1e4d7761b465aa8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732416387449 2024-11-24T02:46:32,798 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d021ba2f3045fea19fde27b64c87d6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732416387449 2024-11-24T02:46:32,799 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9ea05663764653854647038bba7c2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732416389205 2024-11-24T02:46:32,799 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0087702e7c6140189ce539f11458a39c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732416389205 2024-11-24T02:46:32,799 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d83386230c1f4d37805541dff9d4c00f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416390366 2024-11-24T02:46:32,800 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e439cf7005f145dfbc07cee90598f1c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416390366 2024-11-24T02:46:32,811 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:32,811 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/f1f5206f03b944f99e7a898c2365bf24 is 50, key is test_row_0/A:col10/1732416390370/Put/seqid=0 2024-11-24T02:46:32,813 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:32,814 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/3636315c03484f7f941243470ec9e857 is 50, key is test_row_0/B:col10/1732416390370/Put/seqid=0 2024-11-24T02:46:32,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741924_1100 (size=13289) 2024-11-24T02:46:32,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741923_1099 (size=13289) 2024-11-24T02:46:32,839 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/3636315c03484f7f941243470ec9e857 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/3636315c03484f7f941243470ec9e857 2024-11-24T02:46:32,840 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/f1f5206f03b944f99e7a898c2365bf24 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f1f5206f03b944f99e7a898c2365bf24 2024-11-24T02:46:32,854 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 3636315c03484f7f941243470ec9e857(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:32,854 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:32,854 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416392795; duration=0sec 2024-11-24T02:46:32,854 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:32,854 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:32,855 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:32,855 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into f1f5206f03b944f99e7a898c2365bf24(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:32,855 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:32,855 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416392795; duration=0sec 2024-11-24T02:46:32,855 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:32,855 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:32,857 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:32,857 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:32,857 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:32,858 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5cf055a92cfa42eead34205c056b8b01, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/03b4924fe18a44c2a0e2ec06dc44dca1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/40743b85eae241069a7900a1e3b9ddeb] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=36.9 K 2024-11-24T02:46:32,858 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf055a92cfa42eead34205c056b8b01, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732416387449 2024-11-24T02:46:32,859 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b4924fe18a44c2a0e2ec06dc44dca1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732416389205 2024-11-24T02:46:32,860 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 40743b85eae241069a7900a1e3b9ddeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416390366 2024-11-24T02:46:32,885 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#88 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:32,886 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/a65442ace16043ab9dc749c48ecaeca5 is 50, key is test_row_0/C:col10/1732416390370/Put/seqid=0 2024-11-24T02:46:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741925_1101 (size=13255) 2024-11-24T02:46:32,923 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/a65442ace16043ab9dc749c48ecaeca5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a65442ace16043ab9dc749c48ecaeca5 2024-11-24T02:46:32,933 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into a65442ace16043ab9dc749c48ecaeca5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:32,933 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:32,933 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=13, startTime=1732416392795; duration=0sec 2024-11-24T02:46:32,933 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:32,933 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:33,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T02:46:33,397 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:33,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-24T02:46:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:46:33,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:33,400 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:33,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:46:33,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T02:46:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:33,554 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:46:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/a1c1df70a3684ee5acd3a01301aca2bd is 50, key is test_row_0/A:col10/1732416391553/Put/seqid=0 2024-11-24T02:46:33,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741926_1102 (size=12301) 2024-11-24T02:46:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:33,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:46:33,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:33,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416453725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:33,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:33,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416453726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416453836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416453836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:33,983 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/a1c1df70a3684ee5acd3a01301aca2bd 2024-11-24T02:46:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e38a28b9b175419bb69b941d02a4b73e is 50, key is test_row_0/B:col10/1732416391553/Put/seqid=0 2024-11-24T02:46:33,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741927_1103 (size=12301) 2024-11-24T02:46:33,997 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e38a28b9b175419bb69b941d02a4b73e 2024-11-24T02:46:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/948097f30c444abe9bcb4e4e9abd3be1 is 50, key is test_row_0/C:col10/1732416391553/Put/seqid=0 2024-11-24T02:46:34,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741928_1104 (size=12301) 2024-11-24T02:46:34,012 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/948097f30c444abe9bcb4e4e9abd3be1 2024-11-24T02:46:34,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/a1c1df70a3684ee5acd3a01301aca2bd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a1c1df70a3684ee5acd3a01301aca2bd 2024-11-24T02:46:34,025 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a1c1df70a3684ee5acd3a01301aca2bd, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T02:46:34,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/e38a28b9b175419bb69b941d02a4b73e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e38a28b9b175419bb69b941d02a4b73e 2024-11-24T02:46:34,032 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e38a28b9b175419bb69b941d02a4b73e, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T02:46:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:46:34,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/948097f30c444abe9bcb4e4e9abd3be1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/948097f30c444abe9bcb4e4e9abd3be1 2024-11-24T02:46:34,038 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/948097f30c444abe9bcb4e4e9abd3be1, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T02:46:34,039 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 129703e293b1a78847241fc258ac8730 in 485ms, sequenceid=434, compaction requested=false 2024-11-24T02:46:34,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:34,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:34,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-24T02:46:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-24T02:46:34,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T02:46:34,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 641 msec 2024-11-24T02:46:34,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 647 msec 2024-11-24T02:46:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:34,050 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:34,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T02:46:34,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/835b3760fa5d4f95ab9a4292d3a13da8 is 50, key is test_row_0/A:col10/1732416393721/Put/seqid=0 2024-11-24T02:46:34,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741929_1105 (size=14741) 2024-11-24T02:46:34,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416454096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416454098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416454207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416454207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416454417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416454417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/835b3760fa5d4f95ab9a4292d3a13da8 2024-11-24T02:46:34,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/0d7d95ea5dc64d57abeff5f8cacb364c is 50, key is test_row_0/B:col10/1732416393721/Put/seqid=0 2024-11-24T02:46:34,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741930_1106 (size=12301) 2024-11-24T02:46:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T02:46:34,547 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:34,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-24T02:46:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:46:34,549 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:34,550 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:34,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:46:34,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:46:34,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:34,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:34,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416454726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416454726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:34,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:46:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:34,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:34,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:46:34,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/0d7d95ea5dc64d57abeff5f8cacb364c 2024-11-24T02:46:34,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5b6f8a7ba5084a43aa0328b6c72b1e92 is 50, key is test_row_0/C:col10/1732416393721/Put/seqid=0 2024-11-24T02:46:34,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741931_1107 (size=12301) 2024-11-24T02:46:35,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:46:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:46:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:46:35,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416455236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416455236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5b6f8a7ba5084a43aa0328b6c72b1e92 2024-11-24T02:46:35,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/835b3760fa5d4f95ab9a4292d3a13da8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/835b3760fa5d4f95ab9a4292d3a13da8 2024-11-24T02:46:35,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:46:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/835b3760fa5d4f95ab9a4292d3a13da8, entries=200, sequenceid=449, filesize=14.4 K 2024-11-24T02:46:35,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:35,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/0d7d95ea5dc64d57abeff5f8cacb364c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0d7d95ea5dc64d57abeff5f8cacb364c 2024-11-24T02:46:35,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0d7d95ea5dc64d57abeff5f8cacb364c, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T02:46:35,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/5b6f8a7ba5084a43aa0328b6c72b1e92 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5b6f8a7ba5084a43aa0328b6c72b1e92 2024-11-24T02:46:35,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5b6f8a7ba5084a43aa0328b6c72b1e92, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T02:46:35,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 129703e293b1a78847241fc258ac8730 in 1283ms, sequenceid=449, compaction requested=true 2024-11-24T02:46:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:35,334 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:35,334 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:35,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:35,335 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:35,335 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:35,335 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:35,335 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:35,335 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,335 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,335 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f1f5206f03b944f99e7a898c2365bf24, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a1c1df70a3684ee5acd3a01301aca2bd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/835b3760fa5d4f95ab9a4292d3a13da8] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=39.4 K 2024-11-24T02:46:35,335 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/3636315c03484f7f941243470ec9e857, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e38a28b9b175419bb69b941d02a4b73e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0d7d95ea5dc64d57abeff5f8cacb364c] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=37.0 K 2024-11-24T02:46:35,336 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1f5206f03b944f99e7a898c2365bf24, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416390366 2024-11-24T02:46:35,336 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3636315c03484f7f941243470ec9e857, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416390366 2024-11-24T02:46:35,336 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1c1df70a3684ee5acd3a01301aca2bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732416391553 2024-11-24T02:46:35,336 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e38a28b9b175419bb69b941d02a4b73e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732416391553 2024-11-24T02:46:35,336 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 835b3760fa5d4f95ab9a4292d3a13da8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732416393721 2024-11-24T02:46:35,336 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d7d95ea5dc64d57abeff5f8cacb364c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732416393721 2024-11-24T02:46:35,345 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:35,346 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:35,346 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cac86bb23e574c5daf9db73b873aa3ed is 50, key is test_row_0/A:col10/1732416393721/Put/seqid=0 2024-11-24T02:46:35,346 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/6096dd73c4554182a19d5e5b7cef6a28 is 50, key is test_row_0/B:col10/1732416393721/Put/seqid=0 2024-11-24T02:46:35,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741932_1108 (size=13391) 2024-11-24T02:46:35,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741933_1109 (size=13391) 2024-11-24T02:46:35,365 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/6096dd73c4554182a19d5e5b7cef6a28 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6096dd73c4554182a19d5e5b7cef6a28 2024-11-24T02:46:35,367 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/cac86bb23e574c5daf9db73b873aa3ed as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cac86bb23e574c5daf9db73b873aa3ed 2024-11-24T02:46:35,374 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into 6096dd73c4554182a19d5e5b7cef6a28(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:35,374 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into cac86bb23e574c5daf9db73b873aa3ed(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:35,374 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:35,374 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:35,374 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416395334; duration=0sec 2024-11-24T02:46:35,374 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416395334; duration=0sec 2024-11-24T02:46:35,375 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:35,375 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:35,375 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:35,375 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:35,375 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:35,377 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:35,378 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:35,378 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,378 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a65442ace16043ab9dc749c48ecaeca5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/948097f30c444abe9bcb4e4e9abd3be1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5b6f8a7ba5084a43aa0328b6c72b1e92] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=37.0 K 2024-11-24T02:46:35,378 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a65442ace16043ab9dc749c48ecaeca5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416390366 2024-11-24T02:46:35,379 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948097f30c444abe9bcb4e4e9abd3be1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732416391553 2024-11-24T02:46:35,379 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6f8a7ba5084a43aa0328b6c72b1e92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732416393721 2024-11-24T02:46:35,391 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:35,392 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/ad89a0ccda5541999bac4b2c5189ec40 is 50, key is test_row_0/C:col10/1732416393721/Put/seqid=0 2024-11-24T02:46:35,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741934_1110 (size=13357) 2024-11-24T02:46:35,408 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/ad89a0ccda5541999bac4b2c5189ec40 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ad89a0ccda5541999bac4b2c5189ec40 2024-11-24T02:46:35,415 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into ad89a0ccda5541999bac4b2c5189ec40(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:35,415 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:35,415 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=13, startTime=1732416395334; duration=0sec 2024-11-24T02:46:35,416 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:35,416 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:35,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T02:46:35,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:35,470 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:46:35,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d722fc34665f4cb6816d5ff2134a2e14 is 50, key is test_row_0/A:col10/1732416394087/Put/seqid=0 2024-11-24T02:46:35,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741935_1111 (size=12301) 2024-11-24T02:46:35,484 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d722fc34665f4cb6816d5ff2134a2e14 2024-11-24T02:46:35,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/2d76d39be7f94de798a6234e5bab44fe is 50, key is test_row_0/B:col10/1732416394087/Put/seqid=0 2024-11-24T02:46:35,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741936_1112 (size=12301) 2024-11-24T02:46:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:35,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. as already flushing 2024-11-24T02:46:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:46:35,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416455685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416455686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416455687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416455796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416455796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,798 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-24T02:46:35,798 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-24T02:46:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:35,798 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:35,798 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416455797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:35,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:35,799 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-24T02:46:35,799 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:35,801 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-24T02:46:35,801 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:35,803 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:35,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:35,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:35,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:35,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:35,915 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/2d76d39be7f94de798a6234e5bab44fe 2024-11-24T02:46:35,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/6bc6a0aee1cd4287ab816c288149e487 is 50, key is test_row_0/C:col10/1732416394087/Put/seqid=0 2024-11-24T02:46:35,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741937_1113 (size=12301) 2024-11-24T02:46:36,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416456007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416456007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416456007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37160 deadline: 1732416456257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37136 deadline: 1732416456257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,259 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:36,259 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 129703e293b1a78847241fc258ac8730 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37210 deadline: 1732416456317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37162 deadline: 1732416456317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37208 deadline: 1732416456318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:46:36,336 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/6bc6a0aee1cd4287ab816c288149e487 2024-11-24T02:46:36,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/d722fc34665f4cb6816d5ff2134a2e14 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d722fc34665f4cb6816d5ff2134a2e14 2024-11-24T02:46:36,346 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d722fc34665f4cb6816d5ff2134a2e14, entries=150, sequenceid=474, filesize=12.0 K 2024-11-24T02:46:36,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/2d76d39be7f94de798a6234e5bab44fe as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/2d76d39be7f94de798a6234e5bab44fe 2024-11-24T02:46:36,351 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/2d76d39be7f94de798a6234e5bab44fe, entries=150, sequenceid=474, filesize=12.0 K 2024-11-24T02:46:36,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/6bc6a0aee1cd4287ab816c288149e487 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6bc6a0aee1cd4287ab816c288149e487 2024-11-24T02:46:36,356 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6bc6a0aee1cd4287ab816c288149e487, entries=150, sequenceid=474, filesize=12.0 K 2024-11-24T02:46:36,357 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 129703e293b1a78847241fc258ac8730 in 887ms, sequenceid=474, compaction requested=false 2024-11-24T02:46:36,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:36,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:36,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-24T02:46:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-24T02:46:36,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T02:46:36,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-11-24T02:46:36,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8130 sec 2024-11-24T02:46:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T02:46:36,698 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:36,834 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:36,834 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-24T02:46:36,834 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-24T02:46:36,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-24T02:46:36,834 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-24T02:46:36,834 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:36,834 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:36,834 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:36,834 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:36,834 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:36,834 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:36,834 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:36,834 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:36,834 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:36,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:36,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:36,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:36,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/59da2521cad84117abc5c6d7619b3e93 is 50, key is test_row_0/A:col10/1732416396829/Put/seqid=0 2024-11-24T02:46:36,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741938_1114 (size=12301) 2024-11-24T02:46:37,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/59da2521cad84117abc5c6d7619b3e93 2024-11-24T02:46:37,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/8bce6a628a79495ab183afdbd369f165 is 50, key is test_row_0/B:col10/1732416396829/Put/seqid=0 2024-11-24T02:46:37,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741939_1115 (size=12301) 2024-11-24T02:46:37,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/8bce6a628a79495ab183afdbd369f165 2024-11-24T02:46:37,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/d6ab05730afc4aed9c84cf38624def62 is 50, key is test_row_0/C:col10/1732416396829/Put/seqid=0 2024-11-24T02:46:37,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741940_1116 (size=12301) 2024-11-24T02:46:38,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/d6ab05730afc4aed9c84cf38624def62 2024-11-24T02:46:38,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/59da2521cad84117abc5c6d7619b3e93 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/59da2521cad84117abc5c6d7619b3e93 2024-11-24T02:46:38,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/59da2521cad84117abc5c6d7619b3e93, entries=150, sequenceid=491, filesize=12.0 K 2024-11-24T02:46:38,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/8bce6a628a79495ab183afdbd369f165 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/8bce6a628a79495ab183afdbd369f165 2024-11-24T02:46:38,105 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:46:38,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/8bce6a628a79495ab183afdbd369f165, entries=150, sequenceid=491, filesize=12.0 K 2024-11-24T02:46:38,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/d6ab05730afc4aed9c84cf38624def62 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d6ab05730afc4aed9c84cf38624def62 2024-11-24T02:46:38,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d6ab05730afc4aed9c84cf38624def62, entries=150, sequenceid=491, filesize=12.0 K 2024-11-24T02:46:38,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 129703e293b1a78847241fc258ac8730 in 1277ms, sequenceid=491, compaction requested=true 2024-11-24T02:46:38,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:38,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:38,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:38,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:38,112 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:38,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:38,112 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:38,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 129703e293b1a78847241fc258ac8730:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:38,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:38,113 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:38,113 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:38,113 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/B is initiating minor compaction (all files) 2024-11-24T02:46:38,113 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/A is initiating minor compaction (all files) 2024-11-24T02:46:38,113 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/A in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:38,113 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/B in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:38,113 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cac86bb23e574c5daf9db73b873aa3ed, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d722fc34665f4cb6816d5ff2134a2e14, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/59da2521cad84117abc5c6d7619b3e93] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=37.1 K 2024-11-24T02:46:38,113 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6096dd73c4554182a19d5e5b7cef6a28, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/2d76d39be7f94de798a6234e5bab44fe, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/8bce6a628a79495ab183afdbd369f165] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=37.1 K 2024-11-24T02:46:38,113 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6096dd73c4554182a19d5e5b7cef6a28, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732416393721 2024-11-24T02:46:38,113 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cac86bb23e574c5daf9db73b873aa3ed, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732416393721 2024-11-24T02:46:38,114 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d722fc34665f4cb6816d5ff2134a2e14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732416394087 2024-11-24T02:46:38,114 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d76d39be7f94de798a6234e5bab44fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732416394087 2024-11-24T02:46:38,114 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bce6a628a79495ab183afdbd369f165, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416395686 2024-11-24T02:46:38,114 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59da2521cad84117abc5c6d7619b3e93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416395686 2024-11-24T02:46:38,125 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#B#compaction#105 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:38,125 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#A#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:38,126 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b15bdf55975449fcb59ab14ae8cd1988 is 50, key is test_row_0/B:col10/1732416396829/Put/seqid=0 2024-11-24T02:46:38,126 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/93b9c3a01aac46fa93632a81dfd83585 is 50, key is test_row_0/A:col10/1732416396829/Put/seqid=0 2024-11-24T02:46:38,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741942_1118 (size=13493) 2024-11-24T02:46:38,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741941_1117 (size=13493) 2024-11-24T02:46:38,136 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/93b9c3a01aac46fa93632a81dfd83585 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93b9c3a01aac46fa93632a81dfd83585 2024-11-24T02:46:38,142 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/A of 129703e293b1a78847241fc258ac8730 into 93b9c3a01aac46fa93632a81dfd83585(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:38,142 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:38,142 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/A, priority=13, startTime=1732416398111; duration=0sec 2024-11-24T02:46:38,142 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:38,142 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:A 2024-11-24T02:46:38,142 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:38,143 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:38,143 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 129703e293b1a78847241fc258ac8730/C is initiating minor compaction (all files) 2024-11-24T02:46:38,143 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 129703e293b1a78847241fc258ac8730/C in TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:38,144 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ad89a0ccda5541999bac4b2c5189ec40, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6bc6a0aee1cd4287ab816c288149e487, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d6ab05730afc4aed9c84cf38624def62] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp, totalSize=37.1 K 2024-11-24T02:46:38,144 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad89a0ccda5541999bac4b2c5189ec40, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732416393721 2024-11-24T02:46:38,144 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bc6a0aee1cd4287ab816c288149e487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732416394087 2024-11-24T02:46:38,145 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6ab05730afc4aed9c84cf38624def62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1732416395686 2024-11-24T02:46:38,153 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 129703e293b1a78847241fc258ac8730#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:38,153 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/621ffaaa574a4fbebc080ecc3d741820 is 50, key is test_row_0/C:col10/1732416396829/Put/seqid=0 2024-11-24T02:46:38,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741943_1119 (size=13459) 2024-11-24T02:46:38,268 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-24T02:46:38,268 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:38,269 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:38,269 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:38,269 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:38,287 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-24T02:46:38,287 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:46:38,288 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:38,288 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:38,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3530 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3803 2024-11-24T02:46:38,288 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:46:38,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1626 2024-11-24T02:46:38,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4878 rows 2024-11-24T02:46:38,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1625 2024-11-24T02:46:38,289 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4874 rows 2024-11-24T02:46:38,289 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:46:38,289 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:46:38,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:38,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:38,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:46:38,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:46:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:46:38,301 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416398300"}]},"ts":"1732416398300"} 2024-11-24T02:46:38,303 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:46:38,303 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:46:38,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:46:38,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, UNASSIGN}] 2024-11-24T02:46:38,309 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, UNASSIGN 2024-11-24T02:46:38,310 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=129703e293b1a78847241fc258ac8730, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:38,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, UNASSIGN because future has completed 2024-11-24T02:46:38,312 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:46:38,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:46:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:46:38,471 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:38,471 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:46:38,473 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 129703e293b1a78847241fc258ac8730, disabling compactions & flushes 2024-11-24T02:46:38,473 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:38,543 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/b15bdf55975449fcb59ab14ae8cd1988 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b15bdf55975449fcb59ab14ae8cd1988 2024-11-24T02:46:38,552 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/B of 129703e293b1a78847241fc258ac8730 into b15bdf55975449fcb59ab14ae8cd1988(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:38,552 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:38,553 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/B, priority=13, startTime=1732416398112; duration=0sec 2024-11-24T02:46:38,553 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:38,553 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:B 2024-11-24T02:46:38,565 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/621ffaaa574a4fbebc080ecc3d741820 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/621ffaaa574a4fbebc080ecc3d741820 2024-11-24T02:46:38,572 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 129703e293b1a78847241fc258ac8730/C of 129703e293b1a78847241fc258ac8730 into 621ffaaa574a4fbebc080ecc3d741820(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:38,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 129703e293b1a78847241fc258ac8730: 2024-11-24T02:46:38,572 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:38,572 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., storeName=129703e293b1a78847241fc258ac8730/C, priority=13, startTime=1732416398112; duration=0sec 2024-11-24T02:46:38,572 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:38,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:38,572 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. after waiting 0 ms 2024-11-24T02:46:38,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 129703e293b1a78847241fc258ac8730:C 2024-11-24T02:46:38,572 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:38,572 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 129703e293b1a78847241fc258ac8730 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-24T02:46:38,577 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3593a08e634f42dcbdd75d4eba7522c4 is 50, key is test_row_0/A:col10/1732416398286/Put/seqid=0 2024-11-24T02:46:38,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741944_1120 (size=9857) 2024-11-24T02:46:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:46:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:46:38,983 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3593a08e634f42dcbdd75d4eba7522c4 2024-11-24T02:46:38,998 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/fce880e0da874fe9a41e5357189bb6d2 is 50, key is test_row_0/B:col10/1732416398286/Put/seqid=0 2024-11-24T02:46:39,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741945_1121 (size=9857) 2024-11-24T02:46:39,403 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/fce880e0da874fe9a41e5357189bb6d2 2024-11-24T02:46:39,417 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/b7784238d9094e12a6af81afc3223a66 is 50, key is test_row_0/C:col10/1732416398286/Put/seqid=0 2024-11-24T02:46:39,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741946_1122 (size=9857) 2024-11-24T02:46:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:46:39,823 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/b7784238d9094e12a6af81afc3223a66 2024-11-24T02:46:39,835 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/A/3593a08e634f42dcbdd75d4eba7522c4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3593a08e634f42dcbdd75d4eba7522c4 2024-11-24T02:46:39,841 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3593a08e634f42dcbdd75d4eba7522c4, entries=100, sequenceid=499, filesize=9.6 K 2024-11-24T02:46:39,843 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/B/fce880e0da874fe9a41e5357189bb6d2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fce880e0da874fe9a41e5357189bb6d2 2024-11-24T02:46:39,848 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fce880e0da874fe9a41e5357189bb6d2, entries=100, sequenceid=499, filesize=9.6 K 2024-11-24T02:46:39,849 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/.tmp/C/b7784238d9094e12a6af81afc3223a66 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/b7784238d9094e12a6af81afc3223a66 2024-11-24T02:46:39,855 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/b7784238d9094e12a6af81afc3223a66, entries=100, sequenceid=499, filesize=9.6 K 2024-11-24T02:46:39,856 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 129703e293b1a78847241fc258ac8730 in 1283ms, sequenceid=499, compaction requested=false 2024-11-24T02:46:39,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d17437b84b9345c4babaf0296a0342a2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/218b6114b3b6435fafe11ab3cfe27a99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7834cd6d507a4eeaa40d33d7fe8a158a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/afa85aa9fe5d45e5b26014ef943c2055, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04c7743d0c2f4da18db106c4580d71cc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7ded0842617f4123bcbe6f44cfb82b46, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/33bdb74ec4d643bfa67653de0704b708, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d013612cddf44ab8982b0bafffcf6f0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3cb00dba2d5e49b7beb315be4fab0042, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/50ada166cb8a4ed6839dac38dc5c1abf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cc7c5f37b5d042f89d36318efac45adf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/2af94bba5ca64f2bb530cfd3a9d98ff8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/bba3b5658b57414c92421ba648c51814, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4661c816f1254a8eab47177d8afad0f4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d193c17a78b04c41bab4cd27fded561e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f8ff46da541446ee8e2571c1589813d0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3d83c1cb071c41bbacf2de10282e176e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93ad432fffdb409cb7b45ba78a047318, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7de55ad0a92043859051c11a123b686b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/319b6702d25743a5819addd8f0928ef9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f3649f79765643e7bc7e77f1e2c7b71b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/1fd38f73dc454a3fa1b791dba84be68d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d84f76b16c749e5bb1c5c3ec13ca29c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04b3a9b75a2b4f06a888614cdd68d570, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cba15169e6a04161a77b4846ccc163e8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/ad33d93315ba4d8c89563826c2e8e642, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a0d021ba2f3045fea19fde27b64c87d6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/da9ea05663764653854647038bba7c2b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f1f5206f03b944f99e7a898c2365bf24, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d83386230c1f4d37805541dff9d4c00f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a1c1df70a3684ee5acd3a01301aca2bd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/835b3760fa5d4f95ab9a4292d3a13da8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cac86bb23e574c5daf9db73b873aa3ed, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d722fc34665f4cb6816d5ff2134a2e14, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/59da2521cad84117abc5c6d7619b3e93] to archive 2024-11-24T02:46:39,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:46:39,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d17437b84b9345c4babaf0296a0342a2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d17437b84b9345c4babaf0296a0342a2 2024-11-24T02:46:39,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/218b6114b3b6435fafe11ab3cfe27a99 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/218b6114b3b6435fafe11ab3cfe27a99 2024-11-24T02:46:39,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7834cd6d507a4eeaa40d33d7fe8a158a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7834cd6d507a4eeaa40d33d7fe8a158a 2024-11-24T02:46:39,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/afa85aa9fe5d45e5b26014ef943c2055 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/afa85aa9fe5d45e5b26014ef943c2055 2024-11-24T02:46:39,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04c7743d0c2f4da18db106c4580d71cc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04c7743d0c2f4da18db106c4580d71cc 2024-11-24T02:46:39,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7ded0842617f4123bcbe6f44cfb82b46 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7ded0842617f4123bcbe6f44cfb82b46 2024-11-24T02:46:39,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/33bdb74ec4d643bfa67653de0704b708 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/33bdb74ec4d643bfa67653de0704b708 2024-11-24T02:46:39,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d013612cddf44ab8982b0bafffcf6f0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d013612cddf44ab8982b0bafffcf6f0 2024-11-24T02:46:39,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3cb00dba2d5e49b7beb315be4fab0042 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3cb00dba2d5e49b7beb315be4fab0042 2024-11-24T02:46:39,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/50ada166cb8a4ed6839dac38dc5c1abf to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/50ada166cb8a4ed6839dac38dc5c1abf 2024-11-24T02:46:39,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cc7c5f37b5d042f89d36318efac45adf to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cc7c5f37b5d042f89d36318efac45adf 2024-11-24T02:46:39,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/2af94bba5ca64f2bb530cfd3a9d98ff8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/2af94bba5ca64f2bb530cfd3a9d98ff8 2024-11-24T02:46:39,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/bba3b5658b57414c92421ba648c51814 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/bba3b5658b57414c92421ba648c51814 2024-11-24T02:46:39,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4661c816f1254a8eab47177d8afad0f4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4661c816f1254a8eab47177d8afad0f4 2024-11-24T02:46:39,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d193c17a78b04c41bab4cd27fded561e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d193c17a78b04c41bab4cd27fded561e 2024-11-24T02:46:39,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f8ff46da541446ee8e2571c1589813d0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f8ff46da541446ee8e2571c1589813d0 2024-11-24T02:46:39,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3d83c1cb071c41bbacf2de10282e176e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3d83c1cb071c41bbacf2de10282e176e 2024-11-24T02:46:39,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93ad432fffdb409cb7b45ba78a047318 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93ad432fffdb409cb7b45ba78a047318 2024-11-24T02:46:39,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7de55ad0a92043859051c11a123b686b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/7de55ad0a92043859051c11a123b686b 2024-11-24T02:46:39,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/319b6702d25743a5819addd8f0928ef9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/319b6702d25743a5819addd8f0928ef9 2024-11-24T02:46:39,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f3649f79765643e7bc7e77f1e2c7b71b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f3649f79765643e7bc7e77f1e2c7b71b 2024-11-24T02:46:39,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/1fd38f73dc454a3fa1b791dba84be68d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/1fd38f73dc454a3fa1b791dba84be68d 2024-11-24T02:46:39,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d84f76b16c749e5bb1c5c3ec13ca29c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/4d84f76b16c749e5bb1c5c3ec13ca29c 2024-11-24T02:46:39,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04b3a9b75a2b4f06a888614cdd68d570 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/04b3a9b75a2b4f06a888614cdd68d570 2024-11-24T02:46:39,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cba15169e6a04161a77b4846ccc163e8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cba15169e6a04161a77b4846ccc163e8 2024-11-24T02:46:39,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/ad33d93315ba4d8c89563826c2e8e642 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/ad33d93315ba4d8c89563826c2e8e642 2024-11-24T02:46:39,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a0d021ba2f3045fea19fde27b64c87d6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a0d021ba2f3045fea19fde27b64c87d6 2024-11-24T02:46:39,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/da9ea05663764653854647038bba7c2b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/da9ea05663764653854647038bba7c2b 2024-11-24T02:46:39,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f1f5206f03b944f99e7a898c2365bf24 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/f1f5206f03b944f99e7a898c2365bf24 2024-11-24T02:46:39,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d83386230c1f4d37805541dff9d4c00f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d83386230c1f4d37805541dff9d4c00f 2024-11-24T02:46:39,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a1c1df70a3684ee5acd3a01301aca2bd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/a1c1df70a3684ee5acd3a01301aca2bd 2024-11-24T02:46:39,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/835b3760fa5d4f95ab9a4292d3a13da8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/835b3760fa5d4f95ab9a4292d3a13da8 2024-11-24T02:46:39,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cac86bb23e574c5daf9db73b873aa3ed to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/cac86bb23e574c5daf9db73b873aa3ed 2024-11-24T02:46:39,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d722fc34665f4cb6816d5ff2134a2e14 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/d722fc34665f4cb6816d5ff2134a2e14 2024-11-24T02:46:39,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/59da2521cad84117abc5c6d7619b3e93 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/59da2521cad84117abc5c6d7619b3e93 2024-11-24T02:46:39,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/d100c27a2ac044adb0c6c7d7cc25e43b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7ccff740c3047b8b7351ed40de64b8c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7a43a87368c844928aea94b74782bb57, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/80350d3fb0fb4874bfe73563c8db3f1a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/095da2814aec40eb879740514bba2f0e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/13bf2993fd124f7d9a43e00a4d1418ba, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b50b8905ae4e420992e11da284bdc88a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/bbff7b362f6244249109a5cd09a892ee, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6b75bfdb7b164c1d91a7508b6189f36a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7ec5b6f116d5497bb382fcaabf34ae54, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/400d762a95594222a36118b809eb13b6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7d07ae02bb74e73880f9af5396af18b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/adc1b1c976c74cd6a56d9d640c4bebe9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/912ed22665d0492ea05cc90e90806486, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/53043e1d395b4915829405467ad9bbd5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7204bdb21efa426183f28d1407d5eab4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/92da17de6b224ceba9d102a8299d64fd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e0f87769ecf14902b70d3a7ee68ef38c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/443a7519afee4db0bb8d0b3bb00286f3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fcae20d619744950a5948d468930ffa7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e6abc8892fab4ce8b7783bf2a079432b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/796a6875916440e7a6be979086e16284, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/c19f5cbafbe447319715a3c6268bcb06, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/1558b6938a494771a6482ac3469308b1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/937358dcb3014360bb75c24e66d54c7e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/67e00f657f0c428ab1e4d7761b465aa8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e7ad8971a48443af838ab3680025e49a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0087702e7c6140189ce539f11458a39c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/3636315c03484f7f941243470ec9e857, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e439cf7005f145dfbc07cee90598f1c7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e38a28b9b175419bb69b941d02a4b73e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6096dd73c4554182a19d5e5b7cef6a28, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0d7d95ea5dc64d57abeff5f8cacb364c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/2d76d39be7f94de798a6234e5bab44fe, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/8bce6a628a79495ab183afdbd369f165] to archive 2024-11-24T02:46:39,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:46:39,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/d100c27a2ac044adb0c6c7d7cc25e43b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/d100c27a2ac044adb0c6c7d7cc25e43b 2024-11-24T02:46:39,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7ccff740c3047b8b7351ed40de64b8c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7ccff740c3047b8b7351ed40de64b8c 2024-11-24T02:46:39,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7a43a87368c844928aea94b74782bb57 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7a43a87368c844928aea94b74782bb57 2024-11-24T02:46:39,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/80350d3fb0fb4874bfe73563c8db3f1a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/80350d3fb0fb4874bfe73563c8db3f1a 2024-11-24T02:46:39,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/095da2814aec40eb879740514bba2f0e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/095da2814aec40eb879740514bba2f0e 2024-11-24T02:46:39,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/13bf2993fd124f7d9a43e00a4d1418ba to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/13bf2993fd124f7d9a43e00a4d1418ba 2024-11-24T02:46:39,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b50b8905ae4e420992e11da284bdc88a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b50b8905ae4e420992e11da284bdc88a 2024-11-24T02:46:39,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/bbff7b362f6244249109a5cd09a892ee to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/bbff7b362f6244249109a5cd09a892ee 2024-11-24T02:46:39,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6b75bfdb7b164c1d91a7508b6189f36a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6b75bfdb7b164c1d91a7508b6189f36a 2024-11-24T02:46:39,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7ec5b6f116d5497bb382fcaabf34ae54 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7ec5b6f116d5497bb382fcaabf34ae54 2024-11-24T02:46:39,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/400d762a95594222a36118b809eb13b6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/400d762a95594222a36118b809eb13b6 2024-11-24T02:46:39,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7d07ae02bb74e73880f9af5396af18b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b7d07ae02bb74e73880f9af5396af18b 2024-11-24T02:46:39,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/adc1b1c976c74cd6a56d9d640c4bebe9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/adc1b1c976c74cd6a56d9d640c4bebe9 2024-11-24T02:46:39,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/912ed22665d0492ea05cc90e90806486 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/912ed22665d0492ea05cc90e90806486 2024-11-24T02:46:39,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/53043e1d395b4915829405467ad9bbd5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/53043e1d395b4915829405467ad9bbd5 2024-11-24T02:46:39,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7204bdb21efa426183f28d1407d5eab4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/7204bdb21efa426183f28d1407d5eab4 2024-11-24T02:46:39,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/92da17de6b224ceba9d102a8299d64fd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/92da17de6b224ceba9d102a8299d64fd 2024-11-24T02:46:39,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e0f87769ecf14902b70d3a7ee68ef38c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e0f87769ecf14902b70d3a7ee68ef38c 2024-11-24T02:46:39,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/443a7519afee4db0bb8d0b3bb00286f3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/443a7519afee4db0bb8d0b3bb00286f3 2024-11-24T02:46:39,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fcae20d619744950a5948d468930ffa7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fcae20d619744950a5948d468930ffa7 2024-11-24T02:46:39,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e6abc8892fab4ce8b7783bf2a079432b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e6abc8892fab4ce8b7783bf2a079432b 2024-11-24T02:46:39,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/796a6875916440e7a6be979086e16284 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/796a6875916440e7a6be979086e16284 2024-11-24T02:46:39,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/c19f5cbafbe447319715a3c6268bcb06 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/c19f5cbafbe447319715a3c6268bcb06 2024-11-24T02:46:39,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/1558b6938a494771a6482ac3469308b1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/1558b6938a494771a6482ac3469308b1 2024-11-24T02:46:39,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/937358dcb3014360bb75c24e66d54c7e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/937358dcb3014360bb75c24e66d54c7e 2024-11-24T02:46:39,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/67e00f657f0c428ab1e4d7761b465aa8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/67e00f657f0c428ab1e4d7761b465aa8 2024-11-24T02:46:39,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e7ad8971a48443af838ab3680025e49a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e7ad8971a48443af838ab3680025e49a 2024-11-24T02:46:39,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0087702e7c6140189ce539f11458a39c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0087702e7c6140189ce539f11458a39c 2024-11-24T02:46:39,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/3636315c03484f7f941243470ec9e857 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/3636315c03484f7f941243470ec9e857 2024-11-24T02:46:39,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e439cf7005f145dfbc07cee90598f1c7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e439cf7005f145dfbc07cee90598f1c7 2024-11-24T02:46:39,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e38a28b9b175419bb69b941d02a4b73e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/e38a28b9b175419bb69b941d02a4b73e 2024-11-24T02:46:39,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6096dd73c4554182a19d5e5b7cef6a28 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/6096dd73c4554182a19d5e5b7cef6a28 2024-11-24T02:46:39,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0d7d95ea5dc64d57abeff5f8cacb364c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/0d7d95ea5dc64d57abeff5f8cacb364c 2024-11-24T02:46:39,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/2d76d39be7f94de798a6234e5bab44fe to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/2d76d39be7f94de798a6234e5bab44fe 2024-11-24T02:46:39,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/8bce6a628a79495ab183afdbd369f165 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/8bce6a628a79495ab183afdbd369f165 2024-11-24T02:46:39,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/05c680fa246f4bd7ba65c5416c95da05, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/fbaa3270e3f34f588570c7eceeffcf2b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ee3f57bc3f5d4b7cb64117f2737208ee, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/edee406d41f0446b983a82baa823b402, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/93640ef446d041a5bfbf5b39fcccf309, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/4a440f8120764bdeaee62aefcae741d3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a3a4c08dd4b343b7852939166afd0ef8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d57fc132d61045289af613f72967d18e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/7fe9b4908cb24026a116dadac2bdabfb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6fa8ad6be7674cd8b5366f9e5cb0abfb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/2e93986d3ac44cccb3a11a5167e107bd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d9b852d80c144bba4b737681dc75e1e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/011b9eaa128845aabde85f4de7bc0e53, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a6308b2e497c4d6ab3c04c4f20a81644, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/0efe20dc850148b7b02575456e6d054a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cee492f32bcc4400b0b06eb84a659e81, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d88f3ed26ef482887b62de1090e97c1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/1d259267c0c94bed972543cdfffa60f7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5ded04b6768f4100ad0157af344fe87c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/41af04c46a7f4280b80367d4f35d425d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/67fd31aed34c458baab58e799c11c55a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/78f93771cb4a4f40b2fbf5f48aca4330, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cc417039ec30464f9b2810728e11064e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/64e06251ff004e448d0c2ff9ae76bc87, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5cf055a92cfa42eead34205c056b8b01, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/11304f31808d47309c487b2b36c0bceb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/03b4924fe18a44c2a0e2ec06dc44dca1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a65442ace16043ab9dc749c48ecaeca5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/40743b85eae241069a7900a1e3b9ddeb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/948097f30c444abe9bcb4e4e9abd3be1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ad89a0ccda5541999bac4b2c5189ec40, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5b6f8a7ba5084a43aa0328b6c72b1e92, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6bc6a0aee1cd4287ab816c288149e487, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d6ab05730afc4aed9c84cf38624def62] to archive 2024-11-24T02:46:39,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:46:39,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/05c680fa246f4bd7ba65c5416c95da05 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/05c680fa246f4bd7ba65c5416c95da05 2024-11-24T02:46:39,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/fbaa3270e3f34f588570c7eceeffcf2b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/fbaa3270e3f34f588570c7eceeffcf2b 2024-11-24T02:46:39,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ee3f57bc3f5d4b7cb64117f2737208ee to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ee3f57bc3f5d4b7cb64117f2737208ee 2024-11-24T02:46:39,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/edee406d41f0446b983a82baa823b402 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/edee406d41f0446b983a82baa823b402 2024-11-24T02:46:39,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/93640ef446d041a5bfbf5b39fcccf309 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/93640ef446d041a5bfbf5b39fcccf309 2024-11-24T02:46:39,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/4a440f8120764bdeaee62aefcae741d3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/4a440f8120764bdeaee62aefcae741d3 2024-11-24T02:46:39,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a3a4c08dd4b343b7852939166afd0ef8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a3a4c08dd4b343b7852939166afd0ef8 2024-11-24T02:46:39,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d57fc132d61045289af613f72967d18e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d57fc132d61045289af613f72967d18e 2024-11-24T02:46:39,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/7fe9b4908cb24026a116dadac2bdabfb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/7fe9b4908cb24026a116dadac2bdabfb 2024-11-24T02:46:39,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6fa8ad6be7674cd8b5366f9e5cb0abfb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6fa8ad6be7674cd8b5366f9e5cb0abfb 2024-11-24T02:46:39,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/2e93986d3ac44cccb3a11a5167e107bd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/2e93986d3ac44cccb3a11a5167e107bd 2024-11-24T02:46:39,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d9b852d80c144bba4b737681dc75e1e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d9b852d80c144bba4b737681dc75e1e 2024-11-24T02:46:39,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/011b9eaa128845aabde85f4de7bc0e53 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/011b9eaa128845aabde85f4de7bc0e53 2024-11-24T02:46:39,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a6308b2e497c4d6ab3c04c4f20a81644 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a6308b2e497c4d6ab3c04c4f20a81644 2024-11-24T02:46:39,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/0efe20dc850148b7b02575456e6d054a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/0efe20dc850148b7b02575456e6d054a 2024-11-24T02:46:39,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cee492f32bcc4400b0b06eb84a659e81 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cee492f32bcc4400b0b06eb84a659e81 2024-11-24T02:46:39,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d88f3ed26ef482887b62de1090e97c1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/3d88f3ed26ef482887b62de1090e97c1 2024-11-24T02:46:39,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/1d259267c0c94bed972543cdfffa60f7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/1d259267c0c94bed972543cdfffa60f7 2024-11-24T02:46:39,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5ded04b6768f4100ad0157af344fe87c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5ded04b6768f4100ad0157af344fe87c 2024-11-24T02:46:39,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/41af04c46a7f4280b80367d4f35d425d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/41af04c46a7f4280b80367d4f35d425d 2024-11-24T02:46:39,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/67fd31aed34c458baab58e799c11c55a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/67fd31aed34c458baab58e799c11c55a 2024-11-24T02:46:39,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/78f93771cb4a4f40b2fbf5f48aca4330 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/78f93771cb4a4f40b2fbf5f48aca4330 2024-11-24T02:46:39,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cc417039ec30464f9b2810728e11064e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/cc417039ec30464f9b2810728e11064e 2024-11-24T02:46:39,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/64e06251ff004e448d0c2ff9ae76bc87 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/64e06251ff004e448d0c2ff9ae76bc87 2024-11-24T02:46:39,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5cf055a92cfa42eead34205c056b8b01 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5cf055a92cfa42eead34205c056b8b01 2024-11-24T02:46:39,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/11304f31808d47309c487b2b36c0bceb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/11304f31808d47309c487b2b36c0bceb 2024-11-24T02:46:39,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/03b4924fe18a44c2a0e2ec06dc44dca1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/03b4924fe18a44c2a0e2ec06dc44dca1 2024-11-24T02:46:39,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a65442ace16043ab9dc749c48ecaeca5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/a65442ace16043ab9dc749c48ecaeca5 2024-11-24T02:46:39,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/40743b85eae241069a7900a1e3b9ddeb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/40743b85eae241069a7900a1e3b9ddeb 2024-11-24T02:46:39,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/948097f30c444abe9bcb4e4e9abd3be1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/948097f30c444abe9bcb4e4e9abd3be1 2024-11-24T02:46:39,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ad89a0ccda5541999bac4b2c5189ec40 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/ad89a0ccda5541999bac4b2c5189ec40 2024-11-24T02:46:39,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5b6f8a7ba5084a43aa0328b6c72b1e92 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/5b6f8a7ba5084a43aa0328b6c72b1e92 2024-11-24T02:46:39,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6bc6a0aee1cd4287ab816c288149e487 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/6bc6a0aee1cd4287ab816c288149e487 2024-11-24T02:46:39,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d6ab05730afc4aed9c84cf38624def62 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/d6ab05730afc4aed9c84cf38624def62 2024-11-24T02:46:40,003 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/recovered.edits/502.seqid, newMaxSeqId=502, maxSeqId=1 2024-11-24T02:46:40,004 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. 2024-11-24T02:46:40,004 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 129703e293b1a78847241fc258ac8730: Waiting for close lock at 1732416398472Running coprocessor pre-close hooks at 1732416398472Disabling compacts and flushes for region at 1732416398472Disabling writes for close at 1732416398572 (+100 ms)Obtaining lock to block concurrent updates at 1732416398572Preparing flush snapshotting stores in 129703e293b1a78847241fc258ac8730 at 1732416398572Finished memstore snapshotting TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732416398573 (+1 ms)Flushing stores of TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730. at 1732416398573Flushing 129703e293b1a78847241fc258ac8730/A: creating writer at 1732416398573Flushing 129703e293b1a78847241fc258ac8730/A: appending metadata at 1732416398576 (+3 ms)Flushing 129703e293b1a78847241fc258ac8730/A: closing flushed file at 1732416398576Flushing 129703e293b1a78847241fc258ac8730/B: creating writer at 1732416398993 (+417 ms)Flushing 129703e293b1a78847241fc258ac8730/B: appending metadata at 1732416398997 (+4 ms)Flushing 129703e293b1a78847241fc258ac8730/B: closing flushed file at 1732416398997Flushing 129703e293b1a78847241fc258ac8730/C: creating writer at 1732416399413 (+416 ms)Flushing 129703e293b1a78847241fc258ac8730/C: appending metadata at 1732416399417 (+4 ms)Flushing 129703e293b1a78847241fc258ac8730/C: closing flushed file at 1732416399417Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6047a705: reopening flushed file at 1732416399834 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@95865dd: reopening flushed file at 1732416399842 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6153ee03: reopening flushed file at 1732416399848 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 129703e293b1a78847241fc258ac8730 in 1283ms, sequenceid=499, compaction requested=false at 1732416399856 (+8 ms)Writing region close event to WAL at 1732416399999 (+143 ms)Running coprocessor post-close hooks at 1732416400004 (+5 ms)Closed at 1732416400004 2024-11-24T02:46:40,007 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 129703e293b1a78847241fc258ac8730 2024-11-24T02:46:40,008 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=129703e293b1a78847241fc258ac8730, regionState=CLOSED 2024-11-24T02:46:40,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:46:40,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T02:46:40,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 129703e293b1a78847241fc258ac8730, server=7c69a60bd8f6,39461,1732416369827 in 1.6980 sec 2024-11-24T02:46:40,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T02:46:40,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=129703e293b1a78847241fc258ac8730, UNASSIGN in 1.7040 sec 2024-11-24T02:46:40,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T02:46:40,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7110 sec 2024-11-24T02:46:40,018 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416400018"}]},"ts":"1732416400018"} 2024-11-24T02:46:40,019 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:46:40,019 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:46:40,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7260 sec 2024-11-24T02:46:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T02:46:40,448 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:40,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:46:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:40,461 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:46:40,463 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:40,468 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730 2024-11-24T02:46:40,473 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/recovered.edits] 2024-11-24T02:46:40,476 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3593a08e634f42dcbdd75d4eba7522c4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/3593a08e634f42dcbdd75d4eba7522c4 2024-11-24T02:46:40,478 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93b9c3a01aac46fa93632a81dfd83585 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/A/93b9c3a01aac46fa93632a81dfd83585 2024-11-24T02:46:40,481 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b15bdf55975449fcb59ab14ae8cd1988 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/b15bdf55975449fcb59ab14ae8cd1988 2024-11-24T02:46:40,482 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fce880e0da874fe9a41e5357189bb6d2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/B/fce880e0da874fe9a41e5357189bb6d2 2024-11-24T02:46:40,485 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/621ffaaa574a4fbebc080ecc3d741820 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/621ffaaa574a4fbebc080ecc3d741820 2024-11-24T02:46:40,487 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/b7784238d9094e12a6af81afc3223a66 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/C/b7784238d9094e12a6af81afc3223a66 2024-11-24T02:46:40,490 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/recovered.edits/502.seqid to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730/recovered.edits/502.seqid 2024-11-24T02:46:40,491 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/129703e293b1a78847241fc258ac8730 2024-11-24T02:46:40,491 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:46:40,497 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:40,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T02:46:40,506 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:46:40,532 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:46:40,534 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:40,534 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:46:40,535 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416400534"}]},"ts":"9223372036854775807"} 2024-11-24T02:46:40,539 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:46:40,539 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 129703e293b1a78847241fc258ac8730, NAME => 'TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:46:40,539 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:46:40,539 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416400539"}]},"ts":"9223372036854775807"} 2024-11-24T02:46:40,542 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:46:40,543 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:40,545 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 88 msec 2024-11-24T02:46:40,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T02:46:40,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:46:40,568 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:40,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;7c69a60bd8f6:39461-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2077713680_22 at /127.0.0.1:58030 [Waiting for operation #339] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2077713680_22 at /127.0.0.1:38396 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1601595506_22 at /127.0.0.1:42576 [Waiting for operation #329] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1601595506_22 at /127.0.0.1:38408 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=957 (was 1135), ProcessCount=11 (was 11), AvailableMemoryMB=10515 (was 11786) 2024-11-24T02:46:40,588 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=957, ProcessCount=11, AvailableMemoryMB=10513 2024-11-24T02:46:40,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:46:40,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:46:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:40,592 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:46:40,592 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:40,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-24T02:46:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:46:40,594 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:46:40,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741947_1123 (size=959) 2024-11-24T02:46:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:46:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:46:41,003 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:46:41,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741948_1124 (size=53) 2024-11-24T02:46:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:46:41,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:41,414 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 98fb84d8404fffb712280e1a9e01a3ce, disabling compactions & flushes 2024-11-24T02:46:41,414 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:41,414 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:41,414 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. after waiting 0 ms 2024-11-24T02:46:41,414 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:41,414 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:41,415 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 98fb84d8404fffb712280e1a9e01a3ce: Waiting for close lock at 1732416401413Disabling compacts and flushes for region at 1732416401413Disabling writes for close at 1732416401414 (+1 ms)Writing region close event to WAL at 1732416401414Closed at 1732416401414 2024-11-24T02:46:41,417 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:46:41,418 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416401417"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416401417"}]},"ts":"1732416401417"} 2024-11-24T02:46:41,420 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:46:41,422 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:46:41,422 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416401422"}]},"ts":"1732416401422"} 2024-11-24T02:46:41,425 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:46:41,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, ASSIGN}] 2024-11-24T02:46:41,426 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, ASSIGN 2024-11-24T02:46:41,427 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=false 2024-11-24T02:46:41,578 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:41,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, ASSIGN because future has completed 2024-11-24T02:46:41,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:46:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:46:41,748 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:41,748 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:46:41,748 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,748 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:41,749 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,749 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,750 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,752 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98fb84d8404fffb712280e1a9e01a3ce columnFamilyName A 2024-11-24T02:46:41,752 DEBUG [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:41,753 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(327): Store=98fb84d8404fffb712280e1a9e01a3ce/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:41,753 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,754 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98fb84d8404fffb712280e1a9e01a3ce columnFamilyName B 2024-11-24T02:46:41,754 DEBUG [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:41,754 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(327): Store=98fb84d8404fffb712280e1a9e01a3ce/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:41,754 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,755 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98fb84d8404fffb712280e1a9e01a3ce columnFamilyName C 2024-11-24T02:46:41,755 DEBUG [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:41,756 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(327): Store=98fb84d8404fffb712280e1a9e01a3ce/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:41,756 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,757 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,757 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,758 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,758 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,758 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:46:41,759 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,762 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:46:41,763 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 98fb84d8404fffb712280e1a9e01a3ce; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67979996, jitterRate=0.012980878353118896}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:46:41,763 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:41,764 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 98fb84d8404fffb712280e1a9e01a3ce: Running coprocessor pre-open hook at 1732416401749Writing region info on filesystem at 1732416401749Initializing all the Stores at 1732416401749Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416401749Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416401750 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416401750Cleaning up temporary data from old regions at 1732416401758 (+8 ms)Running coprocessor post-open hooks at 1732416401763 (+5 ms)Region opened successfully at 1732416401763 2024-11-24T02:46:41,764 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., pid=32, masterSystemTime=1732416401737 2024-11-24T02:46:41,766 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:41,766 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:41,767 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:41,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:46:41,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T02:46:41,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 in 186 msec 2024-11-24T02:46:41,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T02:46:41,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, ASSIGN in 346 msec 2024-11-24T02:46:41,774 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:46:41,774 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416401774"}]},"ts":"1732416401774"} 2024-11-24T02:46:41,776 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:46:41,777 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:46:41,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-24T02:46:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-24T02:46:42,737 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:42,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:42,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:42,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:42,743 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:42,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cdd40c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:42,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:42,747 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:42,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f1990d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:42,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:42,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:42,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:46:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,43749,1732416369043 2024-11-24T02:46:42,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@695273e8 2024-11-24T02:46:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:46:42,755 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:46:42,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:46:42,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:46:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T02:46:42,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741949_1125 (size=995) 2024-11-24T02:46:43,183 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-24T02:46:43,184 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-24T02:46:43,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:46:43,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, REOPEN/MOVE}] 2024-11-24T02:46:43,201 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, REOPEN/MOVE 2024-11-24T02:46:43,202 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, REOPEN/MOVE because future has completed 2024-11-24T02:46:43,204 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:46:43,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:46:43,357 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,357 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:46:43,357 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 98fb84d8404fffb712280e1a9e01a3ce, disabling compactions & flushes 2024-11-24T02:46:43,357 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,357 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,357 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. after waiting 0 ms 2024-11-24T02:46:43,357 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,362 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T02:46:43,362 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,362 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 98fb84d8404fffb712280e1a9e01a3ce: Waiting for close lock at 1732416403357Running coprocessor pre-close hooks at 1732416403357Disabling compacts and flushes for region at 1732416403357Disabling writes for close at 1732416403357Writing region close event to WAL at 1732416403358 (+1 ms)Running coprocessor post-close hooks at 1732416403362 (+4 ms)Closed at 1732416403362 2024-11-24T02:46:43,362 WARN [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 98fb84d8404fffb712280e1a9e01a3ce to self. 2024-11-24T02:46:43,365 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,365 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=CLOSED 2024-11-24T02:46:43,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:46:43,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-24T02:46:43,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 in 164 msec 2024-11-24T02:46:43,370 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, REOPEN/MOVE; state=CLOSED, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=true 2024-11-24T02:46:43,521 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, REOPEN/MOVE because future has completed 2024-11-24T02:46:43,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:46:43,679 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,679 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:46:43,680 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,680 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:43,680 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,680 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,684 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,689 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98fb84d8404fffb712280e1a9e01a3ce columnFamilyName A 2024-11-24T02:46:43,691 DEBUG [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:43,692 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(327): Store=98fb84d8404fffb712280e1a9e01a3ce/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:43,692 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,693 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98fb84d8404fffb712280e1a9e01a3ce columnFamilyName B 2024-11-24T02:46:43,693 DEBUG [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:43,694 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(327): Store=98fb84d8404fffb712280e1a9e01a3ce/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:43,694 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,695 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98fb84d8404fffb712280e1a9e01a3ce columnFamilyName C 2024-11-24T02:46:43,695 DEBUG [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:43,696 INFO [StoreOpener-98fb84d8404fffb712280e1a9e01a3ce-1 {}] regionserver.HStore(327): Store=98fb84d8404fffb712280e1a9e01a3ce/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:43,696 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,697 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,698 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,699 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,699 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,700 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:46:43,702 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,703 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 98fb84d8404fffb712280e1a9e01a3ce; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67638692, jitterRate=0.007895052433013916}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:46:43,703 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,704 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 98fb84d8404fffb712280e1a9e01a3ce: Running coprocessor pre-open hook at 1732416403680Writing region info on filesystem at 1732416403680Initializing all the Stores at 1732416403681 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416403681Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416403681Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416403681Cleaning up temporary data from old regions at 1732416403700 (+19 ms)Running coprocessor post-open hooks at 1732416403703 (+3 ms)Region opened successfully at 1732416403704 (+1 ms) 2024-11-24T02:46:43,705 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., pid=37, masterSystemTime=1732416403676 2024-11-24T02:46:43,707 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,707 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=OPEN, openSeqNum=5, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:46:43,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-24T02:46:43,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 in 188 msec 2024-11-24T02:46:43,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T02:46:43,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, REOPEN/MOVE in 513 msec 2024-11-24T02:46:43,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-24T02:46:43,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-24T02:46:43,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 959 msec 2024-11-24T02:46:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T02:46:43,727 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:43,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2113fbe8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,732 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d767205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,738 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30259b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,741 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578c795e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,743 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c723f16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,746 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aea187a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,749 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bef9df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562982a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1482606b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,757 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d6a5b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe9caa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,766 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453f7124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,768 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a6519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,771 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d661940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,773 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739a3af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:46:43,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:46:43,775 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:46:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:46:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:46:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439741af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:46:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:46:43,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:46:43,777 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:46:43,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1358217d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:46:43,778 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,780 DEBUG [Thread-630 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,780 DEBUG [Thread-630 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,788 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,791 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,792 DEBUG [Thread-643 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,793 DEBUG [Thread-643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,798 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,800 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,801 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,802 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,804 DEBUG [Thread-641 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,804 DEBUG [Thread-641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,811 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,811 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:46:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,813 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-24T02:46:43,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,815 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:43,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:43,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:46:43,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:43,818 DEBUG [Thread-634 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:46:43,818 DEBUG [Thread-634 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:46:43,819 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:43,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:43,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:46:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:46:43,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416463868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:43,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:43,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416463870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:43,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d8511e0e6a9f49c09b72dca952f5f0c3_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416403813/Put/seqid=0 2024-11-24T02:46:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416463877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416463878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:43,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416463878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:43,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741950_1126 (size=17034) 2024-11-24T02:46:43,910 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:43,916 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d8511e0e6a9f49c09b72dca952f5f0c3_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d8511e0e6a9f49c09b72dca952f5f0c3_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:43,919 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/25d1b16c556e4f5a83ef48a8a16feb17, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:43,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/25d1b16c556e4f5a83ef48a8a16feb17 is 175, key is test_row_0/A:col10/1732416403813/Put/seqid=0 2024-11-24T02:46:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:46:43,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741951_1127 (size=48135) 2024-11-24T02:46:43,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/25d1b16c556e4f5a83ef48a8a16feb17 2024-11-24T02:46:43,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:43,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:43,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:43,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416463987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:43,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416463987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416463996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416463996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416464007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/77811302add84ade86313aef32a70805 is 50, key is test_row_0/B:col10/1732416403813/Put/seqid=0 2024-11-24T02:46:44,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741952_1128 (size=9657) 2024-11-24T02:46:44,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/77811302add84ade86313aef32a70805 2024-11-24T02:46:44,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/84c3559965b0447fb3e6dbeb5be96be9 is 50, key is test_row_0/C:col10/1732416403813/Put/seqid=0 2024-11-24T02:46:44,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:44,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:46:44,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741953_1129 (size=9657) 2024-11-24T02:46:44,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/84c3559965b0447fb3e6dbeb5be96be9 2024-11-24T02:46:44,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/25d1b16c556e4f5a83ef48a8a16feb17 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/25d1b16c556e4f5a83ef48a8a16feb17 2024-11-24T02:46:44,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/25d1b16c556e4f5a83ef48a8a16feb17, entries=250, sequenceid=15, filesize=47.0 K 2024-11-24T02:46:44,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/77811302add84ade86313aef32a70805 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/77811302add84ade86313aef32a70805 2024-11-24T02:46:44,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/77811302add84ade86313aef32a70805, entries=100, sequenceid=15, filesize=9.4 K 2024-11-24T02:46:44,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/84c3559965b0447fb3e6dbeb5be96be9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/84c3559965b0447fb3e6dbeb5be96be9 2024-11-24T02:46:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416464208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416464208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416464208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/84c3559965b0447fb3e6dbeb5be96be9, entries=100, sequenceid=15, filesize=9.4 K 2024-11-24T02:46:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416464209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 98fb84d8404fffb712280e1a9e01a3ce in 402ms, sequenceid=15, compaction requested=false 2024-11-24T02:46:44,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:44,234 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:44,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:46:44,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e2cf4d42b414fbb9e8855b2161e146a_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416404232/Put/seqid=0 2024-11-24T02:46:44,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:44,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741954_1130 (size=14594) 2024-11-24T02:46:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416464294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,304 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:44,310 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e2cf4d42b414fbb9e8855b2161e146a_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e2cf4d42b414fbb9e8855b2161e146a_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:44,311 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4fe91d12b35c4fb59416e89a493d471d, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:44,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4fe91d12b35c4fb59416e89a493d471d is 175, key is test_row_0/A:col10/1732416404232/Put/seqid=0 2024-11-24T02:46:44,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741955_1131 (size=39549) 2024-11-24T02:46:44,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416464407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:44,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:44,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:46:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416464517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416464517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416464527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416464527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416464627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:44,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:44,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4fe91d12b35c4fb59416e89a493d471d 2024-11-24T02:46:44,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f75fc015ed704ebaa5e9a077850522c4 is 50, key is test_row_0/B:col10/1732416404232/Put/seqid=0 2024-11-24T02:46:44,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741956_1132 (size=12001) 2024-11-24T02:46:44,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f75fc015ed704ebaa5e9a077850522c4 2024-11-24T02:46:44,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/ce90368dfa784c048639cd2325af3dc3 is 50, key is test_row_0/C:col10/1732416404232/Put/seqid=0 2024-11-24T02:46:44,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741957_1133 (size=12001) 2024-11-24T02:46:44,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:44,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:44,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:44,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:44,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416464936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:44,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:46:45,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416465026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416465027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:45,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:45,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416465047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:45,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416465047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:45,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:45,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:45,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:45,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:45,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:45,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/ce90368dfa784c048639cd2325af3dc3 2024-11-24T02:46:45,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4fe91d12b35c4fb59416e89a493d471d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4fe91d12b35c4fb59416e89a493d471d 2024-11-24T02:46:45,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4fe91d12b35c4fb59416e89a493d471d, entries=200, sequenceid=42, filesize=38.6 K 2024-11-24T02:46:45,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f75fc015ed704ebaa5e9a077850522c4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f75fc015ed704ebaa5e9a077850522c4 2024-11-24T02:46:45,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f75fc015ed704ebaa5e9a077850522c4, entries=150, sequenceid=42, filesize=11.7 K 2024-11-24T02:46:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/ce90368dfa784c048639cd2325af3dc3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ce90368dfa784c048639cd2325af3dc3 2024-11-24T02:46:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ce90368dfa784c048639cd2325af3dc3, entries=150, sequenceid=42, filesize=11.7 K 2024-11-24T02:46:45,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 98fb84d8404fffb712280e1a9e01a3ce in 1093ms, sequenceid=42, compaction requested=false 2024-11-24T02:46:45,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-24T02:46:45,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:45,379 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-24T02:46:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2c122d62f9145169695418fdfb760fe_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416404267/Put/seqid=0 2024-11-24T02:46:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741958_1134 (size=9714) 2024-11-24T02:46:45,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,451 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2c122d62f9145169695418fdfb760fe_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2c122d62f9145169695418fdfb760fe_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/0b97c8463f934364aa38539ccf4e88cb, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:45,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/0b97c8463f934364aa38539ccf4e88cb is 175, key is test_row_0/A:col10/1732416404267/Put/seqid=0 2024-11-24T02:46:45,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741959_1135 (size=22361) 2024-11-24T02:46:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,564 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:46:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:45,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416465762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:45,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416465876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:45,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:45,881 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/0b97c8463f934364aa38539ccf4e88cb 2024-11-24T02:46:45,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/ff3b7a4028a2410c9b0b91f3513099d2 is 50, key is test_row_0/B:col10/1732416404267/Put/seqid=0 2024-11-24T02:46:45,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741960_1136 (size=9657) 2024-11-24T02:46:45,919 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/ff3b7a4028a2410c9b0b91f3513099d2 2024-11-24T02:46:45,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/516ed0eb64344847baa59eba3663abae is 50, key is test_row_0/C:col10/1732416404267/Put/seqid=0 2024-11-24T02:46:45,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741961_1137 (size=9657) 2024-11-24T02:46:45,963 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/516ed0eb64344847baa59eba3663abae 2024-11-24T02:46:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:46:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/0b97c8463f934364aa38539ccf4e88cb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/0b97c8463f934364aa38539ccf4e88cb 2024-11-24T02:46:45,980 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/0b97c8463f934364aa38539ccf4e88cb, entries=100, sequenceid=51, filesize=21.8 K 2024-11-24T02:46:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/ff3b7a4028a2410c9b0b91f3513099d2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ff3b7a4028a2410c9b0b91f3513099d2 2024-11-24T02:46:45,989 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ff3b7a4028a2410c9b0b91f3513099d2, entries=100, sequenceid=51, filesize=9.4 K 2024-11-24T02:46:45,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/516ed0eb64344847baa59eba3663abae as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/516ed0eb64344847baa59eba3663abae 2024-11-24T02:46:45,997 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/516ed0eb64344847baa59eba3663abae, entries=100, sequenceid=51, filesize=9.4 K 2024-11-24T02:46:45,998 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 98fb84d8404fffb712280e1a9e01a3ce in 620ms, sequenceid=51, compaction requested=true 2024-11-24T02:46:45,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:45,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:45,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-24T02:46:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-24T02:46:46,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-24T02:46:46,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-24T02:46:46,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.1890 sec 2024-11-24T02:46:46,047 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:46,047 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-24T02:46:46,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416466058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416466060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416466065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416466066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,072 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e0fc59723ed4d98ba8313cb5eab6ef2_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416405737/Put/seqid=0 2024-11-24T02:46:46,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416466087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741962_1138 (size=14594) 2024-11-24T02:46:46,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416466167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416466167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416466177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416466377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416466376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416466387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416466407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,511 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:46,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e0fc59723ed4d98ba8313cb5eab6ef2_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e0fc59723ed4d98ba8313cb5eab6ef2_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:46,527 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/5bb7b307ea814ea9be95bf699575fc66, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:46,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/5bb7b307ea814ea9be95bf699575fc66 is 175, key is test_row_0/A:col10/1732416405737/Put/seqid=0 2024-11-24T02:46:46,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741963_1139 (size=39549) 2024-11-24T02:46:46,575 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/5bb7b307ea814ea9be95bf699575fc66 2024-11-24T02:46:46,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/b80a91b84a234fb89568feedc02e744f is 50, key is test_row_0/B:col10/1732416405737/Put/seqid=0 2024-11-24T02:46:46,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741964_1140 (size=12001) 2024-11-24T02:46:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416466697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416466697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416466697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416466917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/b80a91b84a234fb89568feedc02e744f 2024-11-24T02:46:47,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/866dcd44edbe4d2a9d2929067e74f760 is 50, key is test_row_0/C:col10/1732416405737/Put/seqid=0 2024-11-24T02:46:47,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741965_1141 (size=12001) 2024-11-24T02:46:47,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/866dcd44edbe4d2a9d2929067e74f760 2024-11-24T02:46:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/5bb7b307ea814ea9be95bf699575fc66 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/5bb7b307ea814ea9be95bf699575fc66 2024-11-24T02:46:47,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/5bb7b307ea814ea9be95bf699575fc66, entries=200, sequenceid=79, filesize=38.6 K 2024-11-24T02:46:47,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/b80a91b84a234fb89568feedc02e744f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b80a91b84a234fb89568feedc02e744f 2024-11-24T02:46:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b80a91b84a234fb89568feedc02e744f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T02:46:47,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/866dcd44edbe4d2a9d2929067e74f760 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/866dcd44edbe4d2a9d2929067e74f760 2024-11-24T02:46:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/866dcd44edbe4d2a9d2929067e74f760, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T02:46:47,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 98fb84d8404fffb712280e1a9e01a3ce in 1056ms, sequenceid=79, compaction requested=true 2024-11-24T02:46:47,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:47,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:47,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:47,103 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:47,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:47,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:47,103 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:47,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,106 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:47,106 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:47,106 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:46:47,106 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:46:47,106 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:47,106 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:47,106 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/25d1b16c556e4f5a83ef48a8a16feb17, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4fe91d12b35c4fb59416e89a493d471d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/0b97c8463f934364aa38539ccf4e88cb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/5bb7b307ea814ea9be95bf699575fc66] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=146.1 K 2024-11-24T02:46:47,106 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/77811302add84ade86313aef32a70805, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f75fc015ed704ebaa5e9a077850522c4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ff3b7a4028a2410c9b0b91f3513099d2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b80a91b84a234fb89568feedc02e744f] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=42.3 K 2024-11-24T02:46:47,107 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:47,107 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/25d1b16c556e4f5a83ef48a8a16feb17, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4fe91d12b35c4fb59416e89a493d471d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/0b97c8463f934364aa38539ccf4e88cb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/5bb7b307ea814ea9be95bf699575fc66] 2024-11-24T02:46:47,107 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 77811302add84ade86313aef32a70805, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416403803 2024-11-24T02:46:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,108 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f75fc015ed704ebaa5e9a077850522c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732416403867 2024-11-24T02:46:47,108 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25d1b16c556e4f5a83ef48a8a16feb17, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416403793 2024-11-24T02:46:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,108 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting ff3b7a4028a2410c9b0b91f3513099d2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732416404267 2024-11-24T02:46:47,108 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe91d12b35c4fb59416e89a493d471d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732416403867 2024-11-24T02:46:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,109 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b80a91b84a234fb89568feedc02e744f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416405737 2024-11-24T02:46:47,109 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b97c8463f934364aa38539ccf4e88cb, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732416404267 2024-11-24T02:46:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,110 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bb7b307ea814ea9be95bf699575fc66, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416405737 2024-11-24T02:46:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,122 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,134 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,134 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/9a83ee683fa24d20aaec6478294dbc01 is 50, key is test_row_0/B:col10/1732416405737/Put/seqid=0 2024-11-24T02:46:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,141 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112489b509b0c0e94d7ebc2f253e23c50557_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,144 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112489b509b0c0e94d7ebc2f253e23c50557_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:47,144 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112489b509b0c0e94d7ebc2f253e23c50557_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741966_1142 (size=12139) 2024-11-24T02:46:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,180 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/9a83ee683fa24d20aaec6478294dbc01 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/9a83ee683fa24d20aaec6478294dbc01 2024-11-24T02:46:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,186 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into 9a83ee683fa24d20aaec6478294dbc01(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:47,186 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:47,186 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=12, startTime=1732416407103; duration=0sec 2024-11-24T02:46:47,187 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:47,187 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:46:47,187 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,189 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:47,189 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:46:47,189 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:47,189 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/84c3559965b0447fb3e6dbeb5be96be9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ce90368dfa784c048639cd2325af3dc3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/516ed0eb64344847baa59eba3663abae, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/866dcd44edbe4d2a9d2929067e74f760] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=42.3 K 2024-11-24T02:46:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,190 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 84c3559965b0447fb3e6dbeb5be96be9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416403803 2024-11-24T02:46:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,191 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting ce90368dfa784c048639cd2325af3dc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732416403867 2024-11-24T02:46:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,192 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 516ed0eb64344847baa59eba3663abae, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732416404267 2024-11-24T02:46:47,193 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 866dcd44edbe4d2a9d2929067e74f760, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416405737 2024-11-24T02:46:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741967_1143 (size=4469) 2024-11-24T02:46:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,225 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,226 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/162a6559dd754bc09ff96733974f5ee5 is 50, key is test_row_0/C:col10/1732416405737/Put/seqid=0 2024-11-24T02:46:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741968_1144 (size=12139) 2024-11-24T02:46:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:47,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:47,258 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/162a6559dd754bc09ff96733974f5ee5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/162a6559dd754bc09ff96733974f5ee5 2024-11-24T02:46:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fb66b3d79a794c50b453d05bb30e4b16_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416407248/Put/seqid=0 2024-11-24T02:46:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,273 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into 162a6559dd754bc09ff96733974f5ee5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:47,273 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:47,273 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=12, startTime=1732416407103; duration=0sec 2024-11-24T02:46:47,273 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:47,273 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:46:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741969_1145 (size=17034) 2024-11-24T02:46:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,302 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,307 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fb66b3d79a794c50b453d05bb30e4b16_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb66b3d79a794c50b453d05bb30e4b16_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,310 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/41d4c27fc04644e494ecafd139fbf52d, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/41d4c27fc04644e494ecafd139fbf52d is 175, key is test_row_0/A:col10/1732416407248/Put/seqid=0 2024-11-24T02:46:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741970_1146 (size=48135) 2024-11-24T02:46:47,349 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/41d4c27fc04644e494ecafd139fbf52d 2024-11-24T02:46:47,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416467345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416467353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416467356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c94e7d00a9c3441498db165bc96999f5 is 50, key is test_row_0/B:col10/1732416407248/Put/seqid=0 2024-11-24T02:46:47,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741971_1147 (size=9657) 2024-11-24T02:46:47,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c94e7d00a9c3441498db165bc96999f5 2024-11-24T02:46:47,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/46fab24121604e38b5325bdbb0b694d7 is 50, key is test_row_0/C:col10/1732416407248/Put/seqid=0 2024-11-24T02:46:47,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741972_1148 (size=9657) 2024-11-24T02:46:47,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/46fab24121604e38b5325bdbb0b694d7 2024-11-24T02:46:47,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/41d4c27fc04644e494ecafd139fbf52d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/41d4c27fc04644e494ecafd139fbf52d 2024-11-24T02:46:47,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/41d4c27fc04644e494ecafd139fbf52d, entries=250, sequenceid=91, filesize=47.0 K 2024-11-24T02:46:47,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c94e7d00a9c3441498db165bc96999f5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c94e7d00a9c3441498db165bc96999f5 2024-11-24T02:46:47,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416467467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416467467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416467467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c94e7d00a9c3441498db165bc96999f5, entries=100, sequenceid=91, filesize=9.4 K 2024-11-24T02:46:47,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/46fab24121604e38b5325bdbb0b694d7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/46fab24121604e38b5325bdbb0b694d7 2024-11-24T02:46:47,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/46fab24121604e38b5325bdbb0b694d7, entries=100, sequenceid=91, filesize=9.4 K 2024-11-24T02:46:47,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 98fb84d8404fffb712280e1a9e01a3ce in 238ms, sequenceid=91, compaction requested=false 2024-11-24T02:46:47,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:47,616 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#122 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:47,617 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/ec9e87b499954cbdac55b66dcf83c333 is 175, key is test_row_0/A:col10/1732416405737/Put/seqid=0 2024-11-24T02:46:47,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741973_1149 (size=31093) 2024-11-24T02:46:47,645 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/ec9e87b499954cbdac55b66dcf83c333 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ec9e87b499954cbdac55b66dcf83c333 2024-11-24T02:46:47,657 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into ec9e87b499954cbdac55b66dcf83c333(size=30.4 K), total size for store is 77.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:47,657 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:47,657 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=12, startTime=1732416407103; duration=0sec 2024-11-24T02:46:47,659 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:47,659 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:46:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:47,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:47,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:46:47,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124020ee39eb90f4e98ae9f37c861aafedd_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416407681/Put/seqid=0 2024-11-24T02:46:47,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416467699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416467705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416467714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741974_1150 (size=19474) 2024-11-24T02:46:47,740 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,745 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124020ee39eb90f4e98ae9f37c861aafedd_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124020ee39eb90f4e98ae9f37c861aafedd_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:47,747 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/ef058b8678af45958bab7860631bf7bb, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:47,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/ef058b8678af45958bab7860631bf7bb is 175, key is test_row_0/A:col10/1732416407681/Put/seqid=0 2024-11-24T02:46:47,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741975_1151 (size=56733) 2024-11-24T02:46:47,781 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/ef058b8678af45958bab7860631bf7bb 2024-11-24T02:46:47,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c218ac3328864bdb93f947ed1b61343a is 50, key is test_row_0/B:col10/1732416407681/Put/seqid=0 2024-11-24T02:46:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416467817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416467817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416467828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741976_1152 (size=12001) 2024-11-24T02:46:47,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:47,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416467947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:47,952 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:47,953 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:47,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-24T02:46:47,977 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:47,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-24T02:46:47,982 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:47,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:46:47,982 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:47,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:48,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416468027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416468029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416468037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:46:48,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416468096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,097 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:46:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c218ac3328864bdb93f947ed1b61343a 2024-11-24T02:46:48,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/56519688df1b4fa294bac4cee810e261 is 50, key is test_row_0/C:col10/1732416407681/Put/seqid=0 2024-11-24T02:46:48,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:46:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:46:48,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741977_1153 (size=12001) 2024-11-24T02:46:48,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/56519688df1b4fa294bac4cee810e261 2024-11-24T02:46:48,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/ef058b8678af45958bab7860631bf7bb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ef058b8678af45958bab7860631bf7bb 2024-11-24T02:46:48,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ef058b8678af45958bab7860631bf7bb, entries=300, sequenceid=121, filesize=55.4 K 2024-11-24T02:46:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c218ac3328864bdb93f947ed1b61343a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c218ac3328864bdb93f947ed1b61343a 2024-11-24T02:46:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c218ac3328864bdb93f947ed1b61343a, entries=150, sequenceid=121, filesize=11.7 K 2024-11-24T02:46:48,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/56519688df1b4fa294bac4cee810e261 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/56519688df1b4fa294bac4cee810e261 2024-11-24T02:46:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/56519688df1b4fa294bac4cee810e261, entries=150, sequenceid=121, filesize=11.7 K 2024-11-24T02:46:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 98fb84d8404fffb712280e1a9e01a3ce in 659ms, sequenceid=121, compaction requested=true 2024-11-24T02:46:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:48,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:48,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:48,341 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:48,341 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:48,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:48,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:48,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,344 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135961 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:48,344 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:46:48,344 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,344 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ec9e87b499954cbdac55b66dcf83c333, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/41d4c27fc04644e494ecafd139fbf52d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ef058b8678af45958bab7860631bf7bb] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=132.8 K 2024-11-24T02:46:48,344 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,344 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ec9e87b499954cbdac55b66dcf83c333, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/41d4c27fc04644e494ecafd139fbf52d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ef058b8678af45958bab7860631bf7bb] 2024-11-24T02:46:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,345 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec9e87b499954cbdac55b66dcf83c333, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416405737 2024-11-24T02:46:48,345 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:48,345 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:46:48,345 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,345 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d4c27fc04644e494ecafd139fbf52d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732416406058 2024-11-24T02:46:48,346 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/9a83ee683fa24d20aaec6478294dbc01, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c94e7d00a9c3441498db165bc96999f5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c218ac3328864bdb93f947ed1b61343a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=33.0 K 2024-11-24T02:46:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,346 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef058b8678af45958bab7860631bf7bb, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732416407337 2024-11-24T02:46:48,346 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a83ee683fa24d20aaec6478294dbc01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416405737 2024-11-24T02:46:48,347 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c94e7d00a9c3441498db165bc96999f5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732416407248 2024-11-24T02:46:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,347 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c218ac3328864bdb93f947ed1b61343a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732416407351 2024-11-24T02:46:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,366 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#131 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:48,367 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/a4b568b6d6db43c9a341f2cc42a73463 is 50, key is test_row_0/B:col10/1732416407681/Put/seqid=0 2024-11-24T02:46:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,369 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,373 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124f4217d510d084efcad03732a1f1cc8a4_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741978_1154 (size=12241) 2024-11-24T02:46:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,386 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124f4217d510d084efcad03732a1f1cc8a4_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:48,387 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f4217d510d084efcad03732a1f1cc8a4_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,392 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/a4b568b6d6db43c9a341f2cc42a73463 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/a4b568b6d6db43c9a341f2cc42a73463 2024-11-24T02:46:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,399 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into a4b568b6d6db43c9a341f2cc42a73463(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:48,399 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:48,399 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=13, startTime=1732416408341; duration=0sec 2024-11-24T02:46:48,399 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:48,399 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:46:48,399 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,400 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:48,400 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:46:48,400 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,401 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/162a6559dd754bc09ff96733974f5ee5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/46fab24121604e38b5325bdbb0b694d7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/56519688df1b4fa294bac4cee810e261] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=33.0 K 2024-11-24T02:46:48,401 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 162a6559dd754bc09ff96733974f5ee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416405737 2024-11-24T02:46:48,401 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 46fab24121604e38b5325bdbb0b694d7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732416407248 2024-11-24T02:46:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,402 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 56519688df1b4fa294bac4cee810e261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732416407351 2024-11-24T02:46:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741979_1155 (size=4469) 2024-11-24T02:46:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,415 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#132 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:48,416 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/7fc67e5773e046adab66d2073462b774 is 175, key is test_row_0/A:col10/1732416407681/Put/seqid=0 2024-11-24T02:46:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,426 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#133 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:48,427 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/96769e1291c2481c9136acb2c136d782 is 50, key is test_row_0/C:col10/1732416407681/Put/seqid=0 2024-11-24T02:46:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741980_1156 (size=31195) 2024-11-24T02:46:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,442 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/7fc67e5773e046adab66d2073462b774 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/7fc67e5773e046adab66d2073462b774 2024-11-24T02:46:48,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-24T02:46:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,447 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T02:46:48,453 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into 7fc67e5773e046adab66d2073462b774(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:48,453 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:48,453 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=13, startTime=1732416408340; duration=0sec 2024-11-24T02:46:48,453 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:48,453 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:46:48,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741981_1157 (size=12241) 2024-11-24T02:46:48,464 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/96769e1291c2481c9136acb2c136d782 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/96769e1291c2481c9136acb2c136d782 2024-11-24T02:46:48,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fdb13d1beadd44c1aa3e886e67c0e011_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416408446/Put/seqid=0 2024-11-24T02:46:48,483 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into 96769e1291c2481c9136acb2c136d782(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:48,483 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:48,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:48,485 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=13, startTime=1732416408341; duration=0sec 2024-11-24T02:46:48,485 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:48,486 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:46:48,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741982_1158 (size=12204) 2024-11-24T02:46:48,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,522 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fdb13d1beadd44c1aa3e886e67c0e011_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fdb13d1beadd44c1aa3e886e67c0e011_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:48,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/725f4294120b4ca391dfae3732cb57e5, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:48,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/725f4294120b4ca391dfae3732cb57e5 is 175, key is test_row_0/A:col10/1732416408446/Put/seqid=0 2024-11-24T02:46:48,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741983_1159 (size=31005) 2024-11-24T02:46:48,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/725f4294120b4ca391dfae3732cb57e5 2024-11-24T02:46:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/7382367e78884a88a4abbab84d581490 is 50, key is test_row_0/B:col10/1732416408446/Put/seqid=0 2024-11-24T02:46:48,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741984_1160 (size=12051) 2024-11-24T02:46:48,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/7382367e78884a88a4abbab84d581490 2024-11-24T02:46:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/d45fda496bbf4ea69d3a5e34245eb1b0 is 50, key is test_row_0/C:col10/1732416408446/Put/seqid=0 2024-11-24T02:46:48,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741985_1161 (size=12051) 2024-11-24T02:46:48,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:46:48,607 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/d45fda496bbf4ea69d3a5e34245eb1b0 2024-11-24T02:46:48,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/725f4294120b4ca391dfae3732cb57e5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/725f4294120b4ca391dfae3732cb57e5 2024-11-24T02:46:48,627 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/725f4294120b4ca391dfae3732cb57e5, entries=150, sequenceid=131, filesize=30.3 K 2024-11-24T02:46:48,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/7382367e78884a88a4abbab84d581490 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/7382367e78884a88a4abbab84d581490 2024-11-24T02:46:48,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/7382367e78884a88a4abbab84d581490, entries=150, sequenceid=131, filesize=11.8 K 2024-11-24T02:46:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/d45fda496bbf4ea69d3a5e34245eb1b0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d45fda496bbf4ea69d3a5e34245eb1b0 2024-11-24T02:46:48,654 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d45fda496bbf4ea69d3a5e34245eb1b0, entries=150, sequenceid=131, filesize=11.8 K 2024-11-24T02:46:48,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=154.31 KB/158010 for 98fb84d8404fffb712280e1a9e01a3ce in 210ms, sequenceid=131, compaction requested=false 2024-11-24T02:46:48,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:48,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:48,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-24T02:46:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:48,658 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:48,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:46:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-24T02:46:48,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124778c3c49411b4255ae22b11c66ecc88b_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416408650/Put/seqid=0 2024-11-24T02:46:48,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-24T02:46:48,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 680 msec 2024-11-24T02:46:48,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 694 msec 2024-11-24T02:46:48,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416468671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416468672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416468674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741986_1162 (size=19774) 2024-11-24T02:46:48,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:48,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124778c3c49411b4255ae22b11c66ecc88b_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124778c3c49411b4255ae22b11c66ecc88b_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:48,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/97fdf3bc8ff945249ff69cfc42894b2b, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:48,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/97fdf3bc8ff945249ff69cfc42894b2b is 175, key is test_row_0/A:col10/1732416408650/Put/seqid=0 2024-11-24T02:46:48,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741987_1163 (size=57033) 2024-11-24T02:46:48,699 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/97fdf3bc8ff945249ff69cfc42894b2b 2024-11-24T02:46:48,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/bc5bd56ddf3640a9ae536bc063aa98b7 is 50, key is test_row_0/B:col10/1732416408650/Put/seqid=0 2024-11-24T02:46:48,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741988_1164 (size=12151) 2024-11-24T02:46:48,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/bc5bd56ddf3640a9ae536bc063aa98b7 2024-11-24T02:46:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416468777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/d9af62d07cca45c18700a754b4e3908e is 50, key is test_row_0/C:col10/1732416408650/Put/seqid=0 2024-11-24T02:46:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416468789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416468789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741989_1165 (size=12151) 2024-11-24T02:46:48,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416468986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:48,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416468997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416468999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-24T02:46:49,118 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:49,118 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:49,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-24T02:46:49,121 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:49,122 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:49,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:46:49,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/d9af62d07cca45c18700a754b4e3908e 2024-11-24T02:46:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/97fdf3bc8ff945249ff69cfc42894b2b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/97fdf3bc8ff945249ff69cfc42894b2b 2024-11-24T02:46:49,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/97fdf3bc8ff945249ff69cfc42894b2b, entries=300, sequenceid=160, filesize=55.7 K 2024-11-24T02:46:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/bc5bd56ddf3640a9ae536bc063aa98b7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bc5bd56ddf3640a9ae536bc063aa98b7 2024-11-24T02:46:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:46:49,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bc5bd56ddf3640a9ae536bc063aa98b7, entries=150, sequenceid=160, filesize=11.9 K 2024-11-24T02:46:49,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/d9af62d07cca45c18700a754b4e3908e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d9af62d07cca45c18700a754b4e3908e 2024-11-24T02:46:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d9af62d07cca45c18700a754b4e3908e, entries=150, sequenceid=160, filesize=11.9 K 2024-11-24T02:46:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 98fb84d8404fffb712280e1a9e01a3ce in 580ms, sequenceid=160, compaction requested=true 2024-11-24T02:46:49,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:49,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:49,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:49,239 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:49,239 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,242 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:49,243 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:46:49,243 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,243 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/a4b568b6d6db43c9a341f2cc42a73463, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/7382367e78884a88a4abbab84d581490, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bc5bd56ddf3640a9ae536bc063aa98b7] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=35.6 K 2024-11-24T02:46:49,243 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:49,243 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:46:49,243 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:49,243 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/7fc67e5773e046adab66d2073462b774, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/725f4294120b4ca391dfae3732cb57e5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/97fdf3bc8ff945249ff69cfc42894b2b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=116.4 K 2024-11-24T02:46:49,244 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:49,244 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/7fc67e5773e046adab66d2073462b774, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/725f4294120b4ca391dfae3732cb57e5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/97fdf3bc8ff945249ff69cfc42894b2b] 2024-11-24T02:46:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:49,244 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b568b6d6db43c9a341f2cc42a73463, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732416407351 2024-11-24T02:46:49,245 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc67e5773e046adab66d2073462b774, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732416407351 2024-11-24T02:46:49,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,246 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725f4294120b4ca391dfae3732cb57e5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416407688 2024-11-24T02:46:49,246 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 7382367e78884a88a4abbab84d581490, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416407688 2024-11-24T02:46:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,247 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97fdf3bc8ff945249ff69cfc42894b2b, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732416408543 2024-11-24T02:46:49,247 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5bd56ddf3640a9ae536bc063aa98b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732416408642 2024-11-24T02:46:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,271 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,272 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/e7817bd3d4cd40898faca682decd66bb is 50, key is test_row_0/B:col10/1732416408650/Put/seqid=0 2024-11-24T02:46:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T02:46:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,275 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-24T02:46:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,277 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,282 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124a79adf48dded4c9ab592a077488a6e6c_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,285 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124a79adf48dded4c9ab592a077488a6e6c_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:49,285 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a79adf48dded4c9ab592a077488a6e6c_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741990_1166 (size=12493) 2024-11-24T02:46:49,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,305 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/e7817bd3d4cd40898faca682decd66bb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/e7817bd3d4cd40898faca682decd66bb 2024-11-24T02:46:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:49,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112404ece38f86ff4785a7320b3b402b8da9_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416408672/Put/seqid=0 2024-11-24T02:46:49,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,317 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into e7817bd3d4cd40898faca682decd66bb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:49,318 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:49,318 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=13, startTime=1732416409239; duration=0sec 2024-11-24T02:46:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,318 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:49,318 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:46:49,318 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,323 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:49,323 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:46:49,323 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:49,324 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/96769e1291c2481c9136acb2c136d782, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d45fda496bbf4ea69d3a5e34245eb1b0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d9af62d07cca45c18700a754b4e3908e] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=35.6 K 2024-11-24T02:46:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,324 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 96769e1291c2481c9136acb2c136d782, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732416407351 2024-11-24T02:46:49,325 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d45fda496bbf4ea69d3a5e34245eb1b0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732416407688 2024-11-24T02:46:49,326 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d9af62d07cca45c18700a754b4e3908e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732416408642 2024-11-24T02:46:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741991_1167 (size=4469) 2024-11-24T02:46:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,336 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#141 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,337 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/51916a078c9647c588485c888bb533cd is 175, key is test_row_0/A:col10/1732416408650/Put/seqid=0 2024-11-24T02:46:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,343 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:49,344 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/2f6bf588fcad407298c8cf2dffc120e6 is 50, key is test_row_0/C:col10/1732416408650/Put/seqid=0 2024-11-24T02:46:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741992_1168 (size=9814) 2024-11-24T02:46:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741993_1169 (size=31447) 2024-11-24T02:46:49,385 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/51916a078c9647c588485c888bb533cd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/51916a078c9647c588485c888bb533cd 2024-11-24T02:46:49,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741994_1170 (size=12493) 2024-11-24T02:46:49,425 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into 51916a078c9647c588485c888bb533cd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:49,425 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:49,425 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=13, startTime=1732416409238; duration=0sec 2024-11-24T02:46:49,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,425 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:49,425 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:46:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416469392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416469394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416469422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:46:49,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416469536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416469537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416469537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:46:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416469747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416469747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416469747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:49,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:49,774 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112404ece38f86ff4785a7320b3b402b8da9_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404ece38f86ff4785a7320b3b402b8da9_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:49,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/091009835cc7419ab0cda4b1ce22d3ff, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:49,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/091009835cc7419ab0cda4b1ce22d3ff is 175, key is test_row_0/A:col10/1732416408672/Put/seqid=0 2024-11-24T02:46:49,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741995_1171 (size=22461) 2024-11-24T02:46:49,784 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/091009835cc7419ab0cda4b1ce22d3ff 2024-11-24T02:46:49,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/8dbd65cd7ab84fa3aab134a0bde6c60b is 50, key is test_row_0/B:col10/1732416408672/Put/seqid=0 2024-11-24T02:46:49,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741996_1172 (size=9757) 2024-11-24T02:46:49,831 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/2f6bf588fcad407298c8cf2dffc120e6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/2f6bf588fcad407298c8cf2dffc120e6 2024-11-24T02:46:49,836 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into 2f6bf588fcad407298c8cf2dffc120e6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:49,836 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:49,836 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=13, startTime=1732416409244; duration=0sec 2024-11-24T02:46:49,836 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:49,836 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:46:49,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416469988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:49,993 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:49,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:49,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:50,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416470056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:50,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416470056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:50,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416470056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:50,218 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/8dbd65cd7ab84fa3aab134a0bde6c60b 2024-11-24T02:46:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/43dcaf7af9fb47b1bf3a05c9e5fbc473 is 50, key is test_row_0/C:col10/1732416408672/Put/seqid=0 2024-11-24T02:46:50,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741997_1173 (size=9757) 2024-11-24T02:46:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:46:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416470566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:50,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416470577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416470577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:50,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/43dcaf7af9fb47b1bf3a05c9e5fbc473 2024-11-24T02:46:50,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/091009835cc7419ab0cda4b1ce22d3ff as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/091009835cc7419ab0cda4b1ce22d3ff 2024-11-24T02:46:50,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/091009835cc7419ab0cda4b1ce22d3ff, entries=100, sequenceid=169, filesize=21.9 K 2024-11-24T02:46:50,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/8dbd65cd7ab84fa3aab134a0bde6c60b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/8dbd65cd7ab84fa3aab134a0bde6c60b 2024-11-24T02:46:50,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/8dbd65cd7ab84fa3aab134a0bde6c60b, entries=100, sequenceid=169, filesize=9.5 K 2024-11-24T02:46:50,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/43dcaf7af9fb47b1bf3a05c9e5fbc473 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/43dcaf7af9fb47b1bf3a05c9e5fbc473 2024-11-24T02:46:50,688 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/43dcaf7af9fb47b1bf3a05c9e5fbc473, entries=100, sequenceid=169, filesize=9.5 K 2024-11-24T02:46:50,690 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 98fb84d8404fffb712280e1a9e01a3ce in 1414ms, sequenceid=169, compaction requested=false 2024-11-24T02:46:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:50,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-24T02:46:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-24T02:46:50,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T02:46:50,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-24T02:46:50,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5750 sec 2024-11-24T02:46:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T02:46:51,267 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:51,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-24T02:46:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:46:51,269 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:51,270 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:51,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:46:51,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T02:46:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:51,423 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:46:51,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ec8394e9db8b45f19ab1690c06d400bc_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416409380/Put/seqid=0 2024-11-24T02:46:51,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741998_1174 (size=12304) 2024-11-24T02:46:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:46:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:51,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:51,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416471603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416471607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416471607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416471716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416471717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416471717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:51,851 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ec8394e9db8b45f19ab1690c06d400bc_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ec8394e9db8b45f19ab1690c06d400bc_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:51,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/15df87b554ac41c7be5035d4e390a97d, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/15df87b554ac41c7be5035d4e390a97d is 175, key is test_row_0/A:col10/1732416409380/Put/seqid=0 2024-11-24T02:46:51,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741999_1175 (size=31105) 2024-11-24T02:46:51,890 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/15df87b554ac41c7be5035d4e390a97d 2024-11-24T02:46:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:46:51,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/795398d301264ab2bfe3066f57cd235d is 50, key is test_row_0/B:col10/1732416409380/Put/seqid=0 2024-11-24T02:46:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416471927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416471927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416471927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742000_1176 (size=12151) 2024-11-24T02:46:51,939 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/795398d301264ab2bfe3066f57cd235d 2024-11-24T02:46:51,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/149e7bf11d044fc8a1b3a234be8f331c is 50, key is test_row_0/C:col10/1732416409380/Put/seqid=0 2024-11-24T02:46:51,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742001_1177 (size=12151) 2024-11-24T02:46:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416472126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,129 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416472236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416472236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416472237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,369 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/149e7bf11d044fc8a1b3a234be8f331c 2024-11-24T02:46:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/15df87b554ac41c7be5035d4e390a97d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/15df87b554ac41c7be5035d4e390a97d 2024-11-24T02:46:52,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/15df87b554ac41c7be5035d4e390a97d, entries=150, sequenceid=199, filesize=30.4 K 2024-11-24T02:46:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/795398d301264ab2bfe3066f57cd235d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/795398d301264ab2bfe3066f57cd235d 2024-11-24T02:46:52,388 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/795398d301264ab2bfe3066f57cd235d, entries=150, sequenceid=199, filesize=11.9 K 2024-11-24T02:46:52,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/149e7bf11d044fc8a1b3a234be8f331c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/149e7bf11d044fc8a1b3a234be8f331c 2024-11-24T02:46:52,397 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/149e7bf11d044fc8a1b3a234be8f331c, entries=150, sequenceid=199, filesize=11.9 K 2024-11-24T02:46:52,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 98fb84d8404fffb712280e1a9e01a3ce in 975ms, sequenceid=199, compaction requested=true 2024-11-24T02:46:52,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:52,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:52,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-24T02:46:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-24T02:46:52,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T02:46:52,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1300 sec 2024-11-24T02:46:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:46:52,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.1370 sec 2024-11-24T02:46:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:52,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:52,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:46:52,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124682466c6ab81471a9e3ed655a8f51837_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416411591/Put/seqid=0 2024-11-24T02:46:52,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742002_1178 (size=12304) 2024-11-24T02:46:52,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416472822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416472824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416472827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416472937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416472937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416472937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416473147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416473147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416473147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,171 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:53,176 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124682466c6ab81471a9e3ed655a8f51837_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124682466c6ab81471a9e3ed655a8f51837_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:53,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/2cd6682bdfd343c8b05df40b9679a76e, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:53,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/2cd6682bdfd343c8b05df40b9679a76e is 175, key is test_row_0/A:col10/1732416411591/Put/seqid=0 2024-11-24T02:46:53,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742003_1179 (size=31105) 2024-11-24T02:46:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T02:46:53,417 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:53,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:53,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-24T02:46:53,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:46:53,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:53,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:53,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416473457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416473457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416473457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:46:53,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:46:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:53,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,585 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/2cd6682bdfd343c8b05df40b9679a76e 2024-11-24T02:46:53,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/bd856396bcda471186240517cda3fd65 is 50, key is test_row_0/B:col10/1732416411591/Put/seqid=0 2024-11-24T02:46:53,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742004_1180 (size=12151) 2024-11-24T02:46:53,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:46:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:53,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:53,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:46:53,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:46:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:53,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416473966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416473967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416473967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:54,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/bd856396bcda471186240517cda3fd65 2024-11-24T02:46:54,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416474017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:54,018 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:46:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:54,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:54,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/c4a010eaa54d46b79a81da6fba593a5d is 50, key is test_row_0/C:col10/1732416411591/Put/seqid=0 2024-11-24T02:46:54,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742005_1181 (size=12151) 2024-11-24T02:46:54,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:46:54,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:54,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:46:54,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:46:54,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:54,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:46:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/c4a010eaa54d46b79a81da6fba593a5d 2024-11-24T02:46:54,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/2cd6682bdfd343c8b05df40b9679a76e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/2cd6682bdfd343c8b05df40b9679a76e 2024-11-24T02:46:54,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/2cd6682bdfd343c8b05df40b9679a76e, entries=150, sequenceid=211, filesize=30.4 K 2024-11-24T02:46:54,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/bd856396bcda471186240517cda3fd65 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bd856396bcda471186240517cda3fd65 2024-11-24T02:46:54,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bd856396bcda471186240517cda3fd65, entries=150, sequenceid=211, filesize=11.9 K 2024-11-24T02:46:54,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/c4a010eaa54d46b79a81da6fba593a5d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c4a010eaa54d46b79a81da6fba593a5d 2024-11-24T02:46:54,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c4a010eaa54d46b79a81da6fba593a5d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-24T02:46:54,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 98fb84d8404fffb712280e1a9e01a3ce in 1711ms, sequenceid=211, compaction requested=true 2024-11-24T02:46:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:46:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:54,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-24T02:46:54,459 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:54,459 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:54,462 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 116118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:54,462 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:46:54,462 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:54,462 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,462 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:46:54,462 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,462 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/51916a078c9647c588485c888bb533cd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/091009835cc7419ab0cda4b1ce22d3ff, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/15df87b554ac41c7be5035d4e390a97d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/2cd6682bdfd343c8b05df40b9679a76e] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=113.4 K 2024-11-24T02:46:54,462 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,462 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/2f6bf588fcad407298c8cf2dffc120e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/43dcaf7af9fb47b1bf3a05c9e5fbc473, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/149e7bf11d044fc8a1b3a234be8f331c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c4a010eaa54d46b79a81da6fba593a5d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=45.5 K 2024-11-24T02:46:54,462 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/51916a078c9647c588485c888bb533cd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/091009835cc7419ab0cda4b1ce22d3ff, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/15df87b554ac41c7be5035d4e390a97d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/2cd6682bdfd343c8b05df40b9679a76e] 2024-11-24T02:46:54,463 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51916a078c9647c588485c888bb533cd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732416408642 2024-11-24T02:46:54,463 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f6bf588fcad407298c8cf2dffc120e6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732416408642 2024-11-24T02:46:54,463 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 091009835cc7419ab0cda4b1ce22d3ff, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416408669 2024-11-24T02:46:54,463 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 43dcaf7af9fb47b1bf3a05c9e5fbc473, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416408669 2024-11-24T02:46:54,464 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15df87b554ac41c7be5035d4e390a97d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732416409380 2024-11-24T02:46:54,464 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 149e7bf11d044fc8a1b3a234be8f331c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732416409380 2024-11-24T02:46:54,466 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd6682bdfd343c8b05df40b9679a76e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416411591 2024-11-24T02:46:54,466 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a010eaa54d46b79a81da6fba593a5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416411591 2024-11-24T02:46:54,481 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:54,482 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:54,482 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/11b16e2213c245dcb85d84617d40ca15 is 50, key is test_row_0/C:col10/1732416411591/Put/seqid=0 2024-11-24T02:46:54,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T02:46:54,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,496 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124ad078f4450e94479afa24efba5cbfd3a_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:54,496 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:46:54,499 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124ad078f4450e94479afa24efba5cbfd3a_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:54,499 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ad078f4450e94479afa24efba5cbfd3a_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:54,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742006_1182 (size=12629) 2024-11-24T02:46:54,525 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/11b16e2213c245dcb85d84617d40ca15 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/11b16e2213c245dcb85d84617d40ca15 2024-11-24T02:46:54,532 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into 11b16e2213c245dcb85d84617d40ca15(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:54,532 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:54,532 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=12, startTime=1732416414459; duration=0sec 2024-11-24T02:46:54,532 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:54,532 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:46:54,532 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:46:54,536 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:46:54,536 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:46:54,536 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:54,536 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/e7817bd3d4cd40898faca682decd66bb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/8dbd65cd7ab84fa3aab134a0bde6c60b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/795398d301264ab2bfe3066f57cd235d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bd856396bcda471186240517cda3fd65] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=45.5 K 2024-11-24T02:46:54,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742007_1183 (size=4469) 2024-11-24T02:46:54,538 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e7817bd3d4cd40898faca682decd66bb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732416408642 2024-11-24T02:46:54,538 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dbd65cd7ab84fa3aab134a0bde6c60b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732416408669 2024-11-24T02:46:54,538 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 795398d301264ab2bfe3066f57cd235d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732416409380 2024-11-24T02:46:54,539 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting bd856396bcda471186240517cda3fd65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416411591 2024-11-24T02:46:54,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124271d682bf0c44092bcea2694d692d557_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416412819/Put/seqid=0 2024-11-24T02:46:54,549 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:54,549 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/3e0511106bcc41beb1590659a596f9e6 is 50, key is test_row_0/B:col10/1732416411591/Put/seqid=0 2024-11-24T02:46:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:46:54,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742009_1185 (size=12629) 2024-11-24T02:46:54,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742008_1184 (size=12304) 2024-11-24T02:46:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:54,569 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124271d682bf0c44092bcea2694d692d557_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124271d682bf0c44092bcea2694d692d557_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:54,572 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/3e0511106bcc41beb1590659a596f9e6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/3e0511106bcc41beb1590659a596f9e6 2024-11-24T02:46:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/07d6485421bb4c0aa1218a71ed20b8a6, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/07d6485421bb4c0aa1218a71ed20b8a6 is 175, key is test_row_0/A:col10/1732416412819/Put/seqid=0 2024-11-24T02:46:54,581 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into 3e0511106bcc41beb1590659a596f9e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:54,581 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:54,581 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=12, startTime=1732416414459; duration=0sec 2024-11-24T02:46:54,581 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:54,581 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:46:54,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742010_1186 (size=31105) 2024-11-24T02:46:54,942 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#153 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:54,943 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/6911aa6184bc4a14a3d7ef25cc67cc57 is 175, key is test_row_0/A:col10/1732416411591/Put/seqid=0 2024-11-24T02:46:54,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742011_1187 (size=31583) 2024-11-24T02:46:54,963 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/6911aa6184bc4a14a3d7ef25cc67cc57 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/6911aa6184bc4a14a3d7ef25cc67cc57 2024-11-24T02:46:54,974 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into 6911aa6184bc4a14a3d7ef25cc67cc57(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:54,974 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:54,974 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=12, startTime=1732416414459; duration=0sec 2024-11-24T02:46:54,974 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:54,974 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:46:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:54,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:54,990 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/07d6485421bb4c0aa1218a71ed20b8a6 2024-11-24T02:46:55,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416474998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416474998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f978b651825a420885b7cef2f6b027ce is 50, key is test_row_0/B:col10/1732416412819/Put/seqid=0 2024-11-24T02:46:55,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416475000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742012_1188 (size=12151) 2024-11-24T02:46:55,020 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f978b651825a420885b7cef2f6b027ce 2024-11-24T02:46:55,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/ecffbd5530024d478995b3351ddd1ae1 is 50, key is test_row_0/C:col10/1732416412819/Put/seqid=0 2024-11-24T02:46:55,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742013_1189 (size=12151) 2024-11-24T02:46:55,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416475107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416475107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416475118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,141 INFO [master/7c69a60bd8f6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T02:46:55,141 INFO [master/7c69a60bd8f6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T02:46:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416475316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416475317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416475327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,462 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/ecffbd5530024d478995b3351ddd1ae1 2024-11-24T02:46:55,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/07d6485421bb4c0aa1218a71ed20b8a6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/07d6485421bb4c0aa1218a71ed20b8a6 2024-11-24T02:46:55,483 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/07d6485421bb4c0aa1218a71ed20b8a6, entries=150, sequenceid=235, filesize=30.4 K 2024-11-24T02:46:55,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f978b651825a420885b7cef2f6b027ce as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f978b651825a420885b7cef2f6b027ce 2024-11-24T02:46:55,493 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f978b651825a420885b7cef2f6b027ce, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T02:46:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/ecffbd5530024d478995b3351ddd1ae1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ecffbd5530024d478995b3351ddd1ae1 2024-11-24T02:46:55,502 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ecffbd5530024d478995b3351ddd1ae1, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T02:46:55,504 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 98fb84d8404fffb712280e1a9e01a3ce in 1008ms, sequenceid=235, compaction requested=false 2024-11-24T02:46:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:55,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-24T02:46:55,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-24T02:46:55,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-24T02:46:55,508 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0860 sec 2024-11-24T02:46:55,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0900 sec 2024-11-24T02:46:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T02:46:55,567 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:55,567 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:55,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-24T02:46:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:46:55,570 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:55,571 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:55,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:55,629 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:55,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:46:55,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124472781701e0c4178be140d5f374c99e1_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416415628/Put/seqid=0 2024-11-24T02:46:55,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742014_1190 (size=17284) 2024-11-24T02:46:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:46:55,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416475672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416475676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416475680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:55,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:55,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:55,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:55,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416475787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416475787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416475787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:55,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:55,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:46:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:56,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,069 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:56,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124472781701e0c4178be140d5f374c99e1_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124472781701e0c4178be140d5f374c99e1_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:56,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/78e01ffe2b424ca4a7dcde6c4b2cf227, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:56,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/78e01ffe2b424ca4a7dcde6c4b2cf227 is 175, key is test_row_0/A:col10/1732416415628/Put/seqid=0 2024-11-24T02:46:56,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742015_1191 (size=48389) 2024-11-24T02:46:56,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:46:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416476307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416476317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416476317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,488 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/78e01ffe2b424ca4a7dcde6c4b2cf227 2024-11-24T02:46:56,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/741b06b3ed8c43c7b2968d48c61b004a is 50, key is test_row_0/B:col10/1732416415628/Put/seqid=0 2024-11-24T02:46:56,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:56,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742016_1192 (size=12151) 2024-11-24T02:46:56,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:46:56,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:56,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416476817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416476827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416476827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:56,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:56,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/741b06b3ed8c43c7b2968d48c61b004a 2024-11-24T02:46:56,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/13069bc9060b49208d711366915887dc is 50, key is test_row_0/C:col10/1732416415628/Put/seqid=0 2024-11-24T02:46:56,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742017_1193 (size=12151) 2024-11-24T02:46:56,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:56,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:56,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:57,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:57,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/13069bc9060b49208d711366915887dc 2024-11-24T02:46:57,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/78e01ffe2b424ca4a7dcde6c4b2cf227 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/78e01ffe2b424ca4a7dcde6c4b2cf227 2024-11-24T02:46:57,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/78e01ffe2b424ca4a7dcde6c4b2cf227, entries=250, sequenceid=252, filesize=47.3 K 2024-11-24T02:46:57,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/741b06b3ed8c43c7b2968d48c61b004a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/741b06b3ed8c43c7b2968d48c61b004a 2024-11-24T02:46:57,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/741b06b3ed8c43c7b2968d48c61b004a, entries=150, sequenceid=252, filesize=11.9 K 2024-11-24T02:46:57,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/13069bc9060b49208d711366915887dc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/13069bc9060b49208d711366915887dc 2024-11-24T02:46:57,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/13069bc9060b49208d711366915887dc, entries=150, sequenceid=252, filesize=11.9 K 2024-11-24T02:46:57,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 98fb84d8404fffb712280e1a9e01a3ce in 1774ms, sequenceid=252, compaction requested=true 2024-11-24T02:46:57,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:57,404 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:57,404 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:57,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:57,405 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:57,405 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:46:57,405 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,405 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/6911aa6184bc4a14a3d7ef25cc67cc57, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/07d6485421bb4c0aa1218a71ed20b8a6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/78e01ffe2b424ca4a7dcde6c4b2cf227] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=108.5 K 2024-11-24T02:46:57,405 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,405 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/6911aa6184bc4a14a3d7ef25cc67cc57, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/07d6485421bb4c0aa1218a71ed20b8a6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/78e01ffe2b424ca4a7dcde6c4b2cf227] 2024-11-24T02:46:57,405 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:57,405 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:46:57,406 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,406 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/3e0511106bcc41beb1590659a596f9e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f978b651825a420885b7cef2f6b027ce, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/741b06b3ed8c43c7b2968d48c61b004a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=36.1 K 2024-11-24T02:46:57,406 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6911aa6184bc4a14a3d7ef25cc67cc57, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416411591 2024-11-24T02:46:57,406 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0511106bcc41beb1590659a596f9e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416411591 2024-11-24T02:46:57,406 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d6485421bb4c0aa1218a71ed20b8a6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416412811 2024-11-24T02:46:57,406 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f978b651825a420885b7cef2f6b027ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416412811 2024-11-24T02:46:57,407 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e01ffe2b424ca4a7dcde6c4b2cf227, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416414996 2024-11-24T02:46:57,407 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 741b06b3ed8c43c7b2968d48c61b004a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416414998 2024-11-24T02:46:57,414 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:57,415 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:57,415 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/64605c3be5714d0ebf6e609d9d21fc34 is 50, key is test_row_0/B:col10/1732416415628/Put/seqid=0 2024-11-24T02:46:57,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T02:46:57,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,425 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:46:57,438 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246bda9860b7044292951580b1353eaa47_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:57,441 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246bda9860b7044292951580b1353eaa47_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:57,441 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246bda9860b7044292951580b1353eaa47_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:57,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112405175ccbf8c2408e84c3d8db4caa2e79_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416415678/Put/seqid=0 2024-11-24T02:46:57,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742018_1194 (size=12731) 2024-11-24T02:46:57,460 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/64605c3be5714d0ebf6e609d9d21fc34 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/64605c3be5714d0ebf6e609d9d21fc34 2024-11-24T02:46:57,465 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into 64605c3be5714d0ebf6e609d9d21fc34(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:57,466 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:57,466 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=13, startTime=1732416417404; duration=0sec 2024-11-24T02:46:57,466 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:57,466 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:46:57,466 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:57,468 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:57,468 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:46:57,468 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,468 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/11b16e2213c245dcb85d84617d40ca15, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ecffbd5530024d478995b3351ddd1ae1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/13069bc9060b49208d711366915887dc] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=36.1 K 2024-11-24T02:46:57,469 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 11b16e2213c245dcb85d84617d40ca15, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416411591 2024-11-24T02:46:57,469 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting ecffbd5530024d478995b3351ddd1ae1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416412811 2024-11-24T02:46:57,469 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 13069bc9060b49208d711366915887dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416414998 2024-11-24T02:46:57,480 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:57,480 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/26d771cd70b4402aa739612b5e1bf570 is 50, key is test_row_0/C:col10/1732416415628/Put/seqid=0 2024-11-24T02:46:57,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742020_1196 (size=12454) 2024-11-24T02:46:57,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742019_1195 (size=4469) 2024-11-24T02:46:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742021_1197 (size=12731) 2024-11-24T02:46:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:46:57,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:57,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:57,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416477856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:57,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416477858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:57,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416477859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:57,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:57,887 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#162 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:57,887 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4e05aba722014d5bb2460daebc8aad79 is 175, key is test_row_0/A:col10/1732416415628/Put/seqid=0 2024-11-24T02:46:57,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112405175ccbf8c2408e84c3d8db4caa2e79_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112405175ccbf8c2408e84c3d8db4caa2e79_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:57,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d7431c5e3c8b49b196019d000dd25209, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:57,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d7431c5e3c8b49b196019d000dd25209 is 175, key is test_row_0/A:col10/1732416415678/Put/seqid=0 2024-11-24T02:46:57,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742023_1199 (size=31255) 2024-11-24T02:46:57,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742022_1198 (size=31685) 2024-11-24T02:46:57,899 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d7431c5e3c8b49b196019d000dd25209 2024-11-24T02:46:57,902 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/26d771cd70b4402aa739612b5e1bf570 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/26d771cd70b4402aa739612b5e1bf570 2024-11-24T02:46:57,914 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4e05aba722014d5bb2460daebc8aad79 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4e05aba722014d5bb2460daebc8aad79 2024-11-24T02:46:57,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c31fabf4531548e9b2b5c67195e938e8 is 50, key is test_row_0/B:col10/1732416415678/Put/seqid=0 2024-11-24T02:46:57,923 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into 26d771cd70b4402aa739612b5e1bf570(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:57,923 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:57,923 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=13, startTime=1732416417404; duration=0sec 2024-11-24T02:46:57,923 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:57,923 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:46:57,925 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into 4e05aba722014d5bb2460daebc8aad79(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:57,925 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:57,925 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=13, startTime=1732416417403; duration=0sec 2024-11-24T02:46:57,925 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:57,925 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:46:57,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742024_1200 (size=12301) 2024-11-24T02:46:57,927 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c31fabf4531548e9b2b5c67195e938e8 2024-11-24T02:46:57,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/4774733e4cb04d8093f85e725500dd53 is 50, key is test_row_0/C:col10/1732416415678/Put/seqid=0 2024-11-24T02:46:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742025_1201 (size=12301) 2024-11-24T02:46:57,946 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/4774733e4cb04d8093f85e725500dd53 2024-11-24T02:46:57,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d7431c5e3c8b49b196019d000dd25209 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d7431c5e3c8b49b196019d000dd25209 2024-11-24T02:46:57,958 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d7431c5e3c8b49b196019d000dd25209, entries=150, sequenceid=274, filesize=30.5 K 2024-11-24T02:46:57,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/c31fabf4531548e9b2b5c67195e938e8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c31fabf4531548e9b2b5c67195e938e8 2024-11-24T02:46:57,964 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c31fabf4531548e9b2b5c67195e938e8, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:46:57,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/4774733e4cb04d8093f85e725500dd53 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/4774733e4cb04d8093f85e725500dd53 2024-11-24T02:46:57,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416477966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:57,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416477966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:57,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:57,974 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/4774733e4cb04d8093f85e725500dd53, entries=150, sequenceid=274, filesize=12.0 K 2024-11-24T02:46:57,976 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 98fb84d8404fffb712280e1a9e01a3ce in 550ms, sequenceid=274, compaction requested=false 2024-11-24T02:46:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-24T02:46:57,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-24T02:46:57,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-24T02:46:57,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4050 sec 2024-11-24T02:46:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:57,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:57,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T02:46:57,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4140 sec 2024-11-24T02:46:57,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241844ce9773b14cf5a08b5447d0585e2c_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416417853/Put/seqid=0 2024-11-24T02:46:58,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742026_1202 (size=14994) 2024-11-24T02:46:58,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:58,019 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241844ce9773b14cf5a08b5447d0585e2c_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241844ce9773b14cf5a08b5447d0585e2c_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:58,020 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d30a32192c4a41d3bdbd5a801285d0cf, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:58,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d30a32192c4a41d3bdbd5a801285d0cf is 175, key is test_row_0/A:col10/1732416417853/Put/seqid=0 2024-11-24T02:46:58,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742027_1203 (size=39949) 2024-11-24T02:46:58,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416478061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416478177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416478177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416478177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416478387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d30a32192c4a41d3bdbd5a801285d0cf 2024-11-24T02:46:58,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/fc1fe8fdbff04917b296c95cd0b7bf06 is 50, key is test_row_0/B:col10/1732416417853/Put/seqid=0 2024-11-24T02:46:58,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742028_1204 (size=12301) 2024-11-24T02:46:58,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416478487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416478487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416478696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:58,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/fc1fe8fdbff04917b296c95cd0b7bf06 2024-11-24T02:46:58,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/07969fa5ae074449826779c962aece93 is 50, key is test_row_0/C:col10/1732416417853/Put/seqid=0 2024-11-24T02:46:58,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742029_1205 (size=12301) 2024-11-24T02:46:58,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/07969fa5ae074449826779c962aece93 2024-11-24T02:46:58,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/d30a32192c4a41d3bdbd5a801285d0cf as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d30a32192c4a41d3bdbd5a801285d0cf 2024-11-24T02:46:58,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d30a32192c4a41d3bdbd5a801285d0cf, entries=200, sequenceid=292, filesize=39.0 K 2024-11-24T02:46:58,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/fc1fe8fdbff04917b296c95cd0b7bf06 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/fc1fe8fdbff04917b296c95cd0b7bf06 2024-11-24T02:46:58,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/fc1fe8fdbff04917b296c95cd0b7bf06, entries=150, sequenceid=292, filesize=12.0 K 2024-11-24T02:46:58,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/07969fa5ae074449826779c962aece93 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/07969fa5ae074449826779c962aece93 2024-11-24T02:46:58,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/07969fa5ae074449826779c962aece93, entries=150, sequenceid=292, filesize=12.0 K 2024-11-24T02:46:58,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 98fb84d8404fffb712280e1a9e01a3ce in 943ms, sequenceid=292, compaction requested=true 2024-11-24T02:46:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:46:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:46:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:58,924 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:58,924 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:46:58,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:58,928 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:58,928 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:58,928 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:46:58,928 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:46:58,928 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:58,928 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:58,928 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4e05aba722014d5bb2460daebc8aad79, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d7431c5e3c8b49b196019d000dd25209, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d30a32192c4a41d3bdbd5a801285d0cf] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=100.5 K 2024-11-24T02:46:58,928 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/64605c3be5714d0ebf6e609d9d21fc34, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c31fabf4531548e9b2b5c67195e938e8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/fc1fe8fdbff04917b296c95cd0b7bf06] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=36.5 K 2024-11-24T02:46:58,928 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:58,928 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4e05aba722014d5bb2460daebc8aad79, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d7431c5e3c8b49b196019d000dd25209, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d30a32192c4a41d3bdbd5a801285d0cf] 2024-11-24T02:46:58,928 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 64605c3be5714d0ebf6e609d9d21fc34, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416414998 2024-11-24T02:46:58,928 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e05aba722014d5bb2460daebc8aad79, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416414998 2024-11-24T02:46:58,929 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c31fabf4531548e9b2b5c67195e938e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416415661 2024-11-24T02:46:58,929 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7431c5e3c8b49b196019d000dd25209, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416415661 2024-11-24T02:46:58,930 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting fc1fe8fdbff04917b296c95cd0b7bf06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732416417844 2024-11-24T02:46:58,930 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d30a32192c4a41d3bdbd5a801285d0cf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732416417844 2024-11-24T02:46:58,938 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:58,940 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:58,940 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/530db6947b7747a49502d4b8e1f0f97a is 50, key is test_row_0/B:col10/1732416417853/Put/seqid=0 2024-11-24T02:46:58,949 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124987d3630769542d6a0d288be6f434f3f_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:58,951 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124987d3630769542d6a0d288be6f434f3f_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:58,951 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124987d3630769542d6a0d288be6f434f3f_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:58,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742030_1206 (size=12983) 2024-11-24T02:46:58,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742031_1207 (size=4469) 2024-11-24T02:46:58,968 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#171 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:58,969 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/302a879c5fe24bd0a9f9bb05a337fd1b is 175, key is test_row_0/A:col10/1732416417853/Put/seqid=0 2024-11-24T02:46:58,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742032_1208 (size=31937) 2024-11-24T02:46:59,003 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:46:59,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:46:59,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:59,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112403dea72eba464ade90b75ca90fc77f4d_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416419001/Put/seqid=0 2024-11-24T02:46:59,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416479043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742033_1209 (size=12454) 2024-11-24T02:46:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416479045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416479157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416479157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416479206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,369 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/530db6947b7747a49502d4b8e1f0f97a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/530db6947b7747a49502d4b8e1f0f97a 2024-11-24T02:46:59,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416479367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416479367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,374 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into 530db6947b7747a49502d4b8e1f0f97a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:59,374 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:59,374 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=13, startTime=1732416418924; duration=0sec 2024-11-24T02:46:59,374 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:46:59,374 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:46:59,374 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:46:59,375 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:46:59,375 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:46:59,375 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:59,376 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/26d771cd70b4402aa739612b5e1bf570, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/4774733e4cb04d8093f85e725500dd53, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/07969fa5ae074449826779c962aece93] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=36.5 K 2024-11-24T02:46:59,376 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d771cd70b4402aa739612b5e1bf570, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416414998 2024-11-24T02:46:59,376 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4774733e4cb04d8093f85e725500dd53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732416415661 2024-11-24T02:46:59,377 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 07969fa5ae074449826779c962aece93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732416417844 2024-11-24T02:46:59,384 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:46:59,385 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/66d1769945e340bfb53313eaa0cbfaa9 is 50, key is test_row_0/C:col10/1732416417853/Put/seqid=0 2024-11-24T02:46:59,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742034_1210 (size=12983) 2024-11-24T02:46:59,405 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/302a879c5fe24bd0a9f9bb05a337fd1b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/302a879c5fe24bd0a9f9bb05a337fd1b 2024-11-24T02:46:59,411 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into 302a879c5fe24bd0a9f9bb05a337fd1b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:59,411 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:59,411 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=13, startTime=1732416418924; duration=0sec 2024-11-24T02:46:59,411 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:59,411 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:46:59,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:59,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112403dea72eba464ade90b75ca90fc77f4d_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112403dea72eba464ade90b75ca90fc77f4d_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:46:59,456 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4b5e1be94b554c5a91419e3e319a868c, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:46:59,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4b5e1be94b554c5a91419e3e319a868c is 175, key is test_row_0/A:col10/1732416419001/Put/seqid=0 2024-11-24T02:46:59,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742035_1211 (size=31255) 2024-11-24T02:46:59,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416479676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:46:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416479677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:46:59,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:46:59,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:46:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T02:46:59,726 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:46:59,727 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:46:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-24T02:46:59,729 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:46:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:46:59,729 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:46:59,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:46:59,799 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/66d1769945e340bfb53313eaa0cbfaa9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/66d1769945e340bfb53313eaa0cbfaa9 2024-11-24T02:46:59,804 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into 66d1769945e340bfb53313eaa0cbfaa9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:46:59,804 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:46:59,804 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=13, startTime=1732416418924; duration=0sec 2024-11-24T02:46:59,804 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:46:59,804 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:46:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:46:59,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4b5e1be94b554c5a91419e3e319a868c 2024-11-24T02:46:59,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:46:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:46:59,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:46:59,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/4fb81a39bd14424eb032217f81ddb3f8 is 50, key is test_row_0/B:col10/1732416419001/Put/seqid=0 2024-11-24T02:46:59,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742036_1212 (size=12301) 2024-11-24T02:46:59,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/4fb81a39bd14424eb032217f81ddb3f8 2024-11-24T02:46:59,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/71602a5d684445bd9e1e3e269d60489b is 50, key is test_row_0/C:col10/1732416419001/Put/seqid=0 2024-11-24T02:46:59,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742037_1213 (size=12301) 2024-11-24T02:47:00,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:47:00,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:00,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:00,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:00,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:47:00,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416480187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:00,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416480187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:00,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:00,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:47:00,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:00,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:00,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:00,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416480226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:00,228 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:00,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/71602a5d684445bd9e1e3e269d60489b 2024-11-24T02:47:00,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/4b5e1be94b554c5a91419e3e319a868c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4b5e1be94b554c5a91419e3e319a868c 2024-11-24T02:47:00,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4b5e1be94b554c5a91419e3e319a868c, entries=150, sequenceid=314, filesize=30.5 K 2024-11-24T02:47:00,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/4fb81a39bd14424eb032217f81ddb3f8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/4fb81a39bd14424eb032217f81ddb3f8 2024-11-24T02:47:00,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/4fb81a39bd14424eb032217f81ddb3f8, entries=150, sequenceid=314, filesize=12.0 K 2024-11-24T02:47:00,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/71602a5d684445bd9e1e3e269d60489b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/71602a5d684445bd9e1e3e269d60489b 2024-11-24T02:47:00,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/71602a5d684445bd9e1e3e269d60489b, entries=150, sequenceid=314, filesize=12.0 K 2024-11-24T02:47:00,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 98fb84d8404fffb712280e1a9e01a3ce in 1341ms, sequenceid=314, compaction requested=false 2024-11-24T02:47:00,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:00,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T02:47:00,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:00,349 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:47:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:47:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c6314f755114466c83fa1241eb7437fa_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416419019/Put/seqid=0 2024-11-24T02:47:00,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742038_1214 (size=12454) 2024-11-24T02:47:00,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:00,776 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c6314f755114466c83fa1241eb7437fa_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c6314f755114466c83fa1241eb7437fa_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/f79b790bf80b4fa5b889010e97ffb493, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:00,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/f79b790bf80b4fa5b889010e97ffb493 is 175, key is test_row_0/A:col10/1732416419019/Put/seqid=0 2024-11-24T02:47:00,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742039_1215 (size=31255) 2024-11-24T02:47:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:47:01,184 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/f79b790bf80b4fa5b889010e97ffb493 2024-11-24T02:47:01,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/d16d6b02ca384dbdae2db0d3e3f92fae is 50, key is test_row_0/B:col10/1732416419019/Put/seqid=0 2024-11-24T02:47:01,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742040_1216 (size=12301) 2024-11-24T02:47:01,203 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/d16d6b02ca384dbdae2db0d3e3f92fae 2024-11-24T02:47:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:01,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:01,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/73a4951d6c6e4a21a24cfa1661f49738 is 50, key is test_row_0/C:col10/1732416419019/Put/seqid=0 2024-11-24T02:47:01,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742041_1217 (size=12301) 2024-11-24T02:47:01,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416481277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416481279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416481387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416481387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416481596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416481596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,633 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/73a4951d6c6e4a21a24cfa1661f49738 2024-11-24T02:47:01,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/f79b790bf80b4fa5b889010e97ffb493 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/f79b790bf80b4fa5b889010e97ffb493 2024-11-24T02:47:01,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/f79b790bf80b4fa5b889010e97ffb493, entries=150, sequenceid=331, filesize=30.5 K 2024-11-24T02:47:01,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/d16d6b02ca384dbdae2db0d3e3f92fae as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/d16d6b02ca384dbdae2db0d3e3f92fae 2024-11-24T02:47:01,665 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/d16d6b02ca384dbdae2db0d3e3f92fae, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:47:01,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/73a4951d6c6e4a21a24cfa1661f49738 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/73a4951d6c6e4a21a24cfa1661f49738 2024-11-24T02:47:01,671 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/73a4951d6c6e4a21a24cfa1661f49738, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:47:01,676 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 98fb84d8404fffb712280e1a9e01a3ce in 1326ms, sequenceid=331, compaction requested=true 2024-11-24T02:47:01,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:01,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:01,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T02:47:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T02:47:01,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T02:47:01,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9470 sec 2024-11-24T02:47:01,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.9550 sec 2024-11-24T02:47:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T02:47:01,877 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:01,877 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-24T02:47:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:47:01,880 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:01,880 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:01,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:01,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:01,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T02:47:01,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e5b1b2ad21914f9db58313d229108a44_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416421257/Put/seqid=0 2024-11-24T02:47:01,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416481930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416481936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:01,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742042_1218 (size=14994) 2024-11-24T02:47:01,945 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:01,949 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e5b1b2ad21914f9db58313d229108a44_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e5b1b2ad21914f9db58313d229108a44_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:01,950 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/701ad3263d424399b55de89518b06057, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:01,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/701ad3263d424399b55de89518b06057 is 175, key is test_row_0/A:col10/1732416421257/Put/seqid=0 2024-11-24T02:47:01,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742043_1219 (size=39949) 2024-11-24T02:47:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:47:02,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:47:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:02,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416482046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416482047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:47:02,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60324 deadline: 1732416482197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,198 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18321 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:47:02,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60284 deadline: 1732416482246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,250 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,250 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416482257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416482257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:47:02,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,377 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/701ad3263d424399b55de89518b06057 2024-11-24T02:47:02,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f1ecf947f33a4e27854083ff98089611 is 50, key is test_row_0/B:col10/1732416421257/Put/seqid=0 2024-11-24T02:47:02,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742044_1220 (size=12301) 2024-11-24T02:47:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:47:02,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:47:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:02,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:47:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f1ecf947f33a4e27854083ff98089611 2024-11-24T02:47:02,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/c67379d97f7443a5922abfc3af00c25f is 50, key is test_row_0/C:col10/1732416421257/Put/seqid=0 2024-11-24T02:47:02,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742045_1221 (size=12301) 2024-11-24T02:47:02,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/c67379d97f7443a5922abfc3af00c25f 2024-11-24T02:47:02,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/701ad3263d424399b55de89518b06057 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/701ad3263d424399b55de89518b06057 2024-11-24T02:47:02,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/701ad3263d424399b55de89518b06057, entries=200, sequenceid=354, filesize=39.0 K 2024-11-24T02:47:02,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/f1ecf947f33a4e27854083ff98089611 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f1ecf947f33a4e27854083ff98089611 2024-11-24T02:47:02,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f1ecf947f33a4e27854083ff98089611, entries=150, sequenceid=354, filesize=12.0 K 2024-11-24T02:47:02,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/c67379d97f7443a5922abfc3af00c25f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c67379d97f7443a5922abfc3af00c25f 2024-11-24T02:47:02,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c67379d97f7443a5922abfc3af00c25f, entries=150, sequenceid=354, filesize=12.0 K 2024-11-24T02:47:02,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 98fb84d8404fffb712280e1a9e01a3ce in 921ms, sequenceid=354, compaction requested=true 2024-11-24T02:47:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:02,830 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:02,830 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:02,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:02,831 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:02,831 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:47:02,831 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,832 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/530db6947b7747a49502d4b8e1f0f97a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/4fb81a39bd14424eb032217f81ddb3f8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/d16d6b02ca384dbdae2db0d3e3f92fae, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f1ecf947f33a4e27854083ff98089611] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=48.7 K 2024-11-24T02:47:02,832 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 530db6947b7747a49502d4b8e1f0f97a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732416417844 2024-11-24T02:47:02,832 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:02,832 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:47:02,832 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,833 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/302a879c5fe24bd0a9f9bb05a337fd1b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4b5e1be94b554c5a91419e3e319a868c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/f79b790bf80b4fa5b889010e97ffb493, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/701ad3263d424399b55de89518b06057] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=131.2 K 2024-11-24T02:47:02,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T02:47:02,833 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,833 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/302a879c5fe24bd0a9f9bb05a337fd1b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4b5e1be94b554c5a91419e3e319a868c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/f79b790bf80b4fa5b889010e97ffb493, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/701ad3263d424399b55de89518b06057] 2024-11-24T02:47:02,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,833 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:47:02,833 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fb81a39bd14424eb032217f81ddb3f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732416418003 2024-11-24T02:47:02,834 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d16d6b02ca384dbdae2db0d3e3f92fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416419009 2024-11-24T02:47:02,834 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302a879c5fe24bd0a9f9bb05a337fd1b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732416417844 2024-11-24T02:47:02,834 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ecf947f33a4e27854083ff98089611, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732416421257 2024-11-24T02:47:02,834 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b5e1be94b554c5a91419e3e319a868c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732416418003 2024-11-24T02:47:02,834 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f79b790bf80b4fa5b889010e97ffb493, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416419009 2024-11-24T02:47:02,835 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 701ad3263d424399b55de89518b06057, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732416421249 2024-11-24T02:47:02,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d9e92c73eaab4b23a076d9e50104f6ee_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416421917/Put/seqid=0 2024-11-24T02:47:02,846 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:02,847 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/99225b814fd947f8b250c410d0d7703c is 50, key is test_row_0/B:col10/1732416421257/Put/seqid=0 2024-11-24T02:47:02,853 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:02,865 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124f26f60e60a2949bcbfd11fe5ff57f4dd_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:02,868 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124f26f60e60a2949bcbfd11fe5ff57f4dd_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:02,868 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f26f60e60a2949bcbfd11fe5ff57f4dd_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:02,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742046_1222 (size=12454) 2024-11-24T02:47:02,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:02,879 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d9e92c73eaab4b23a076d9e50104f6ee_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d9e92c73eaab4b23a076d9e50104f6ee_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:02,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/c1a2ff76e3ac4637b3d8bc37292dc639, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:02,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/c1a2ff76e3ac4637b3d8bc37292dc639 is 175, key is test_row_0/A:col10/1732416421917/Put/seqid=0 2024-11-24T02:47:02,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742047_1223 (size=13119) 2024-11-24T02:47:02,897 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/99225b814fd947f8b250c410d0d7703c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/99225b814fd947f8b250c410d0d7703c 2024-11-24T02:47:02,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742048_1224 (size=4469) 2024-11-24T02:47:02,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742049_1225 (size=31255) 2024-11-24T02:47:02,907 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/c1a2ff76e3ac4637b3d8bc37292dc639 2024-11-24T02:47:02,913 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into 99225b814fd947f8b250c410d0d7703c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:02,913 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:02,913 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=12, startTime=1732416422830; duration=0sec 2024-11-24T02:47:02,913 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:02,913 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:47:02,913 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:02,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/21ec06e68ef847b697c0d4f829314887 is 50, key is test_row_0/B:col10/1732416421917/Put/seqid=0 2024-11-24T02:47:02,919 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:02,919 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:47:02,919 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:02,919 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/66d1769945e340bfb53313eaa0cbfaa9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/71602a5d684445bd9e1e3e269d60489b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/73a4951d6c6e4a21a24cfa1661f49738, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c67379d97f7443a5922abfc3af00c25f] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=48.7 K 2024-11-24T02:47:02,920 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d1769945e340bfb53313eaa0cbfaa9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732416417844 2024-11-24T02:47:02,920 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 71602a5d684445bd9e1e3e269d60489b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732416418003 2024-11-24T02:47:02,922 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 73a4951d6c6e4a21a24cfa1661f49738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416419009 2024-11-24T02:47:02,923 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c67379d97f7443a5922abfc3af00c25f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732416421257 2024-11-24T02:47:02,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742050_1226 (size=12301) 2024-11-24T02:47:02,925 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/21ec06e68ef847b697c0d4f829314887 2024-11-24T02:47:02,955 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:02,956 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/204d8427a33741aa91c35481900fe11a is 50, key is test_row_0/C:col10/1732416421257/Put/seqid=0 2024-11-24T02:47:02,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/9c7ff674397e4d168fcda01e30bf7605 is 50, key is test_row_0/C:col10/1732416421917/Put/seqid=0 2024-11-24T02:47:02,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742051_1227 (size=13119) 2024-11-24T02:47:02,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742052_1228 (size=12301) 2024-11-24T02:47:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:47:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:03,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. as already flushing 2024-11-24T02:47:03,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416483136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416483140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416483246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416483257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,304 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#184 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:03,305 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/542be654b1f5401ba572f271e83e2d0d is 175, key is test_row_0/A:col10/1732416421257/Put/seqid=0 2024-11-24T02:47:03,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742053_1229 (size=32073) 2024-11-24T02:47:03,315 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/542be654b1f5401ba572f271e83e2d0d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/542be654b1f5401ba572f271e83e2d0d 2024-11-24T02:47:03,321 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into 542be654b1f5401ba572f271e83e2d0d(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:03,321 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:03,321 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=12, startTime=1732416422830; duration=0sec 2024-11-24T02:47:03,321 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:03,321 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:47:03,382 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/204d8427a33741aa91c35481900fe11a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/204d8427a33741aa91c35481900fe11a 2024-11-24T02:47:03,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/9c7ff674397e4d168fcda01e30bf7605 2024-11-24T02:47:03,389 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into 204d8427a33741aa91c35481900fe11a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:03,389 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:03,389 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=12, startTime=1732416422830; duration=0sec 2024-11-24T02:47:03,389 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:03,389 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:47:03,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/c1a2ff76e3ac4637b3d8bc37292dc639 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/c1a2ff76e3ac4637b3d8bc37292dc639 2024-11-24T02:47:03,397 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/c1a2ff76e3ac4637b3d8bc37292dc639, entries=150, sequenceid=367, filesize=30.5 K 2024-11-24T02:47:03,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/21ec06e68ef847b697c0d4f829314887 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/21ec06e68ef847b697c0d4f829314887 2024-11-24T02:47:03,405 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/21ec06e68ef847b697c0d4f829314887, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T02:47:03,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/9c7ff674397e4d168fcda01e30bf7605 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/9c7ff674397e4d168fcda01e30bf7605 2024-11-24T02:47:03,414 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/9c7ff674397e4d168fcda01e30bf7605, entries=150, sequenceid=367, filesize=12.0 K 2024-11-24T02:47:03,415 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 98fb84d8404fffb712280e1a9e01a3ce in 582ms, sequenceid=367, compaction requested=false 2024-11-24T02:47:03,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:03,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:03,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-24T02:47:03,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-24T02:47:03,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T02:47:03,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5370 sec 2024-11-24T02:47:03,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.5460 sec 2024-11-24T02:47:03,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:03,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:03,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:47:03,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112450973bcf917d412694a9db37003e87a0_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416423133/Put/seqid=0 2024-11-24T02:47:03,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416483485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416483486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742054_1230 (size=14994) 2024-11-24T02:47:03,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:03,504 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112450973bcf917d412694a9db37003e87a0_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112450973bcf917d412694a9db37003e87a0_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:03,505 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/548779c92c29484c9d81e110d9e383da, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:03,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/548779c92c29484c9d81e110d9e383da is 175, key is test_row_0/A:col10/1732416423133/Put/seqid=0 2024-11-24T02:47:03,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742055_1231 (size=39949) 2024-11-24T02:47:03,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416483596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416483596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416483807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416483807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:03,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:03,813 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-24T02:47:03,813 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:03,813 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,813 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,813 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:03,815 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-24T02:47:03,815 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:03,815 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,816 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:03,816 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-24T02:47:03,816 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:03,816 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,816 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:03,821 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-24T02:47:03,821 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:03,821 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,821 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:03,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:03,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/548779c92c29484c9d81e110d9e383da 2024-11-24T02:47:03,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/ed48bae5a3f0496f930d2852e9eb5700 is 50, key is test_row_0/B:col10/1732416423133/Put/seqid=0 2024-11-24T02:47:03,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742056_1232 (size=12301) 2024-11-24T02:47:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T02:47:04,026 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:04,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60258 deadline: 1732416484107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:04,108 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 98fb84d8404fffb712280e1a9e01a3ce of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18346 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:04,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:04,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416484116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416484116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:04,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/ed48bae5a3f0496f930d2852e9eb5700 2024-11-24T02:47:04,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/e89f9d77527c40d4aeaf09d26e7fa39a is 50, key is test_row_0/C:col10/1732416423133/Put/seqid=0 2024-11-24T02:47:04,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742057_1233 (size=12301) 2024-11-24T02:47:04,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:04,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60310 deadline: 1732416484627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60304 deadline: 1732416484627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:04,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:04,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/e89f9d77527c40d4aeaf09d26e7fa39a 2024-11-24T02:47:04,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/548779c92c29484c9d81e110d9e383da as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/548779c92c29484c9d81e110d9e383da 2024-11-24T02:47:04,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/548779c92c29484c9d81e110d9e383da, entries=200, sequenceid=394, filesize=39.0 K 2024-11-24T02:47:04,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/ed48bae5a3f0496f930d2852e9eb5700 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ed48bae5a3f0496f930d2852e9eb5700 2024-11-24T02:47:04,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ed48bae5a3f0496f930d2852e9eb5700, entries=150, sequenceid=394, filesize=12.0 K 2024-11-24T02:47:04,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/e89f9d77527c40d4aeaf09d26e7fa39a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/e89f9d77527c40d4aeaf09d26e7fa39a 2024-11-24T02:47:04,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/e89f9d77527c40d4aeaf09d26e7fa39a, entries=150, sequenceid=394, filesize=12.0 K 2024-11-24T02:47:04,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 98fb84d8404fffb712280e1a9e01a3ce in 1313ms, sequenceid=394, compaction requested=true 2024-11-24T02:47:04,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:04,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:04,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:04,773 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:04,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:04,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:04,773 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:04,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 98fb84d8404fffb712280e1a9e01a3ce:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:04,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:04,774 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:04,774 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:04,774 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/B is initiating minor compaction (all files) 2024-11-24T02:47:04,774 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/A is initiating minor compaction (all files) 2024-11-24T02:47:04,774 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/A in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:04,774 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/B in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:04,774 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/542be654b1f5401ba572f271e83e2d0d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/c1a2ff76e3ac4637b3d8bc37292dc639, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/548779c92c29484c9d81e110d9e383da] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=100.9 K 2024-11-24T02:47:04,774 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/99225b814fd947f8b250c410d0d7703c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/21ec06e68ef847b697c0d4f829314887, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ed48bae5a3f0496f930d2852e9eb5700] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=36.8 K 2024-11-24T02:47:04,774 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:04,774 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/542be654b1f5401ba572f271e83e2d0d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/c1a2ff76e3ac4637b3d8bc37292dc639, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/548779c92c29484c9d81e110d9e383da] 2024-11-24T02:47:04,774 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 99225b814fd947f8b250c410d0d7703c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732416421257 2024-11-24T02:47:04,774 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 542be654b1f5401ba572f271e83e2d0d, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732416421257 2024-11-24T02:47:04,775 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ec06e68ef847b697c0d4f829314887, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732416421917 2024-11-24T02:47:04,775 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a2ff76e3ac4637b3d8bc37292dc639, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732416421917 2024-11-24T02:47:04,775 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting ed48bae5a3f0496f930d2852e9eb5700, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732416423133 2024-11-24T02:47:04,775 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 548779c92c29484c9d81e110d9e383da, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732416423133 2024-11-24T02:47:04,782 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:04,783 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#B#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:04,783 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/0705bdfdff38430baf8540c82d716281 is 50, key is test_row_0/B:col10/1732416423133/Put/seqid=0 2024-11-24T02:47:04,784 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411247fe720d55587475b84fb25afcd8cb4f4_98fb84d8404fffb712280e1a9e01a3ce store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:04,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742058_1234 (size=13221) 2024-11-24T02:47:04,807 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411247fe720d55587475b84fb25afcd8cb4f4_98fb84d8404fffb712280e1a9e01a3ce, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:04,807 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247fe720d55587475b84fb25afcd8cb4f4_98fb84d8404fffb712280e1a9e01a3ce because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:04,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742059_1235 (size=4469) 2024-11-24T02:47:05,193 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/0705bdfdff38430baf8540c82d716281 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/0705bdfdff38430baf8540c82d716281 2024-11-24T02:47:05,200 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/B of 98fb84d8404fffb712280e1a9e01a3ce into 0705bdfdff38430baf8540c82d716281(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:05,200 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:05,200 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/B, priority=13, startTime=1732416424773; duration=0sec 2024-11-24T02:47:05,200 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:05,200 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:B 2024-11-24T02:47:05,200 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:05,201 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:05,201 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 98fb84d8404fffb712280e1a9e01a3ce/C is initiating minor compaction (all files) 2024-11-24T02:47:05,201 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 98fb84d8404fffb712280e1a9e01a3ce/C in TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:05,201 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/204d8427a33741aa91c35481900fe11a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/9c7ff674397e4d168fcda01e30bf7605, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/e89f9d77527c40d4aeaf09d26e7fa39a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp, totalSize=36.8 K 2024-11-24T02:47:05,202 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 204d8427a33741aa91c35481900fe11a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1732416421257 2024-11-24T02:47:05,202 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7ff674397e4d168fcda01e30bf7605, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732416421917 2024-11-24T02:47:05,202 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e89f9d77527c40d4aeaf09d26e7fa39a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1732416423133 2024-11-24T02:47:05,214 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#C#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:05,214 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/f335f4b9dd4f4ac28b2d9a684d23ab44 is 50, key is test_row_0/C:col10/1732416423133/Put/seqid=0 2024-11-24T02:47:05,214 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 98fb84d8404fffb712280e1a9e01a3ce#A#compaction#192 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:05,215 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/23dd602e8bf84bbe920a66324f6cb9b0 is 175, key is test_row_0/A:col10/1732416423133/Put/seqid=0 2024-11-24T02:47:05,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742061_1237 (size=32175) 2024-11-24T02:47:05,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742060_1236 (size=13221) 2024-11-24T02:47:05,628 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/23dd602e8bf84bbe920a66324f6cb9b0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/23dd602e8bf84bbe920a66324f6cb9b0 2024-11-24T02:47:05,628 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/f335f4b9dd4f4ac28b2d9a684d23ab44 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/f335f4b9dd4f4ac28b2d9a684d23ab44 2024-11-24T02:47:05,632 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/C of 98fb84d8404fffb712280e1a9e01a3ce into f335f4b9dd4f4ac28b2d9a684d23ab44(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:05,632 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 98fb84d8404fffb712280e1a9e01a3ce/A of 98fb84d8404fffb712280e1a9e01a3ce into 23dd602e8bf84bbe920a66324f6cb9b0(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:05,632 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:05,632 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:05,632 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/C, priority=13, startTime=1732416424773; duration=0sec 2024-11-24T02:47:05,632 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., storeName=98fb84d8404fffb712280e1a9e01a3ce/A, priority=13, startTime=1732416424772; duration=0sec 2024-11-24T02:47:05,632 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:05,632 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:C 2024-11-24T02:47:05,632 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:05,632 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 98fb84d8404fffb712280e1a9e01a3ce:A 2024-11-24T02:47:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:05,639 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:05,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:47:05,639 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-24T02:47:05,639 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:05,639 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:05,639 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:05,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:05,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244ebda4bb3ce840faa435266fe5b289d5_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416425637/Put/seqid=0 2024-11-24T02:47:05,648 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-24T02:47:05,648 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:05,648 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:05,648 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:05,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742062_1238 (size=12454) 2024-11-24T02:47:06,050 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:06,055 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244ebda4bb3ce840faa435266fe5b289d5_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ebda4bb3ce840faa435266fe5b289d5_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:06,057 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/67870c03e4004236a21cd1b6e39d8549, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:06,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/67870c03e4004236a21cd1b6e39d8549 is 175, key is test_row_0/A:col10/1732416425637/Put/seqid=0 2024-11-24T02:47:06,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742063_1239 (size=31255) 2024-11-24T02:47:06,278 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-24T02:47:06,278 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:06,278 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:06,279 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:06,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:06,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/67870c03e4004236a21cd1b6e39d8549 2024-11-24T02:47:06,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/49a5fc4b4a2d4f6a96028215353bde9d is 50, key is test_row_0/B:col10/1732416425637/Put/seqid=0 2024-11-24T02:47:06,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742064_1240 (size=12301) 2024-11-24T02:47:06,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/49a5fc4b4a2d4f6a96028215353bde9d 2024-11-24T02:47:06,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/29ab4fa5f0a44d9cb27f1b33fef55bad is 50, key is test_row_0/C:col10/1732416425637/Put/seqid=0 2024-11-24T02:47:06,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742065_1241 (size=12301) 2024-11-24T02:47:07,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/29ab4fa5f0a44d9cb27f1b33fef55bad 2024-11-24T02:47:07,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/67870c03e4004236a21cd1b6e39d8549 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/67870c03e4004236a21cd1b6e39d8549 2024-11-24T02:47:07,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/67870c03e4004236a21cd1b6e39d8549, entries=150, sequenceid=410, filesize=30.5 K 2024-11-24T02:47:07,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/49a5fc4b4a2d4f6a96028215353bde9d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/49a5fc4b4a2d4f6a96028215353bde9d 2024-11-24T02:47:07,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/49a5fc4b4a2d4f6a96028215353bde9d, entries=150, sequenceid=410, filesize=12.0 K 2024-11-24T02:47:07,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/29ab4fa5f0a44d9cb27f1b33fef55bad as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/29ab4fa5f0a44d9cb27f1b33fef55bad 2024-11-24T02:47:07,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/29ab4fa5f0a44d9cb27f1b33fef55bad, entries=150, sequenceid=410, filesize=12.0 K 2024-11-24T02:47:07,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for 98fb84d8404fffb712280e1a9e01a3ce in 1679ms, sequenceid=410, compaction requested=false 2024-11-24T02:47:07,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 98fb84d8404fffb712280e1a9e01a3ce: 2024-11-24T02:47:08,105 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:47:12,248 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-24T02:47:12,248 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:12,248 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:12,248 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:12,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:14,169 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-24T02:47:14,169 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:14,169 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:14,169 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:14,169 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 5 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:47:14,169 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3128 2024-11-24T02:47:14,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3182 2024-11-24T02:47:14,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:47:14,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1394 2024-11-24T02:47:14,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4181 rows 2024-11-24T02:47:14,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1398 2024-11-24T02:47:14,170 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4191 rows 2024-11-24T02:47:14,170 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:47:14,170 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:47:14,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:14,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:14,170 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:14,171 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:47:14,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:47:14,174 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416434174"}]},"ts":"1732416434174"} 2024-11-24T02:47:14,176 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:47:14,176 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:47:14,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:47:14,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, UNASSIGN}] 2024-11-24T02:47:14,179 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, UNASSIGN 2024-11-24T02:47:14,180 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:14,182 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, UNASSIGN because future has completed 2024-11-24T02:47:14,182 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:47:14,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:47:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:47:14,335 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:14,335 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:47:14,335 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 98fb84d8404fffb712280e1a9e01a3ce, disabling compactions & flushes 2024-11-24T02:47:14,335 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:14,335 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:14,335 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. after waiting 0 ms 2024-11-24T02:47:14,335 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:14,335 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 98fb84d8404fffb712280e1a9e01a3ce 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-24T02:47:14,344 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f60201caa3f04b4b8ecc9636a1b72528_98fb84d8404fffb712280e1a9e01a3ce is 50, key is test_row_0/A:col10/1732416426277/Put/seqid=0 2024-11-24T02:47:14,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742066_1242 (size=12454) 2024-11-24T02:47:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:47:14,749 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:14,755 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f60201caa3f04b4b8ecc9636a1b72528_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f60201caa3f04b4b8ecc9636a1b72528_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:14,756 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/395177e748ac40ab82c00f475ed93246, store: [table=TestAcidGuarantees family=A region=98fb84d8404fffb712280e1a9e01a3ce] 2024-11-24T02:47:14,757 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/395177e748ac40ab82c00f475ed93246 is 175, key is test_row_0/A:col10/1732416426277/Put/seqid=0 2024-11-24T02:47:14,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742067_1243 (size=31255) 2024-11-24T02:47:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:47:15,163 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=417, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/395177e748ac40ab82c00f475ed93246 2024-11-24T02:47:15,172 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/b5c0eed7f2b04fc0935c8e5a74d95136 is 50, key is test_row_0/B:col10/1732416426277/Put/seqid=0 2024-11-24T02:47:15,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742068_1244 (size=12301) 2024-11-24T02:47:15,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:47:15,577 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/b5c0eed7f2b04fc0935c8e5a74d95136 2024-11-24T02:47:15,586 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/aedf605d7ebf43c2ab31440e12391897 is 50, key is test_row_0/C:col10/1732416426277/Put/seqid=0 2024-11-24T02:47:15,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742069_1245 (size=12301) 2024-11-24T02:47:15,991 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/aedf605d7ebf43c2ab31440e12391897 2024-11-24T02:47:15,996 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/A/395177e748ac40ab82c00f475ed93246 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/395177e748ac40ab82c00f475ed93246 2024-11-24T02:47:16,001 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/395177e748ac40ab82c00f475ed93246, entries=150, sequenceid=417, filesize=30.5 K 2024-11-24T02:47:16,002 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/B/b5c0eed7f2b04fc0935c8e5a74d95136 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b5c0eed7f2b04fc0935c8e5a74d95136 2024-11-24T02:47:16,007 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b5c0eed7f2b04fc0935c8e5a74d95136, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T02:47:16,008 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/.tmp/C/aedf605d7ebf43c2ab31440e12391897 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/aedf605d7ebf43c2ab31440e12391897 2024-11-24T02:47:16,012 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/aedf605d7ebf43c2ab31440e12391897, entries=150, sequenceid=417, filesize=12.0 K 2024-11-24T02:47:16,013 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 98fb84d8404fffb712280e1a9e01a3ce in 1678ms, sequenceid=417, compaction requested=true 2024-11-24T02:47:16,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/25d1b16c556e4f5a83ef48a8a16feb17, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4fe91d12b35c4fb59416e89a493d471d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/0b97c8463f934364aa38539ccf4e88cb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/5bb7b307ea814ea9be95bf699575fc66, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ec9e87b499954cbdac55b66dcf83c333, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/41d4c27fc04644e494ecafd139fbf52d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ef058b8678af45958bab7860631bf7bb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/7fc67e5773e046adab66d2073462b774, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/725f4294120b4ca391dfae3732cb57e5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/97fdf3bc8ff945249ff69cfc42894b2b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/51916a078c9647c588485c888bb533cd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/091009835cc7419ab0cda4b1ce22d3ff, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/15df87b554ac41c7be5035d4e390a97d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/6911aa6184bc4a14a3d7ef25cc67cc57, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/2cd6682bdfd343c8b05df40b9679a76e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/07d6485421bb4c0aa1218a71ed20b8a6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/78e01ffe2b424ca4a7dcde6c4b2cf227, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4e05aba722014d5bb2460daebc8aad79, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d7431c5e3c8b49b196019d000dd25209, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d30a32192c4a41d3bdbd5a801285d0cf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/302a879c5fe24bd0a9f9bb05a337fd1b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4b5e1be94b554c5a91419e3e319a868c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/f79b790bf80b4fa5b889010e97ffb493, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/701ad3263d424399b55de89518b06057, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/542be654b1f5401ba572f271e83e2d0d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/c1a2ff76e3ac4637b3d8bc37292dc639, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/548779c92c29484c9d81e110d9e383da] to archive 2024-11-24T02:47:16,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:47:16,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/25d1b16c556e4f5a83ef48a8a16feb17 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/25d1b16c556e4f5a83ef48a8a16feb17 2024-11-24T02:47:16,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4fe91d12b35c4fb59416e89a493d471d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4fe91d12b35c4fb59416e89a493d471d 2024-11-24T02:47:16,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/0b97c8463f934364aa38539ccf4e88cb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/0b97c8463f934364aa38539ccf4e88cb 2024-11-24T02:47:16,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/5bb7b307ea814ea9be95bf699575fc66 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/5bb7b307ea814ea9be95bf699575fc66 2024-11-24T02:47:16,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ec9e87b499954cbdac55b66dcf83c333 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ec9e87b499954cbdac55b66dcf83c333 2024-11-24T02:47:16,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/41d4c27fc04644e494ecafd139fbf52d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/41d4c27fc04644e494ecafd139fbf52d 2024-11-24T02:47:16,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ef058b8678af45958bab7860631bf7bb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/ef058b8678af45958bab7860631bf7bb 2024-11-24T02:47:16,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/7fc67e5773e046adab66d2073462b774 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/7fc67e5773e046adab66d2073462b774 2024-11-24T02:47:16,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/725f4294120b4ca391dfae3732cb57e5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/725f4294120b4ca391dfae3732cb57e5 2024-11-24T02:47:16,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/97fdf3bc8ff945249ff69cfc42894b2b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/97fdf3bc8ff945249ff69cfc42894b2b 2024-11-24T02:47:16,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/51916a078c9647c588485c888bb533cd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/51916a078c9647c588485c888bb533cd 2024-11-24T02:47:16,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/091009835cc7419ab0cda4b1ce22d3ff to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/091009835cc7419ab0cda4b1ce22d3ff 2024-11-24T02:47:16,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/15df87b554ac41c7be5035d4e390a97d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/15df87b554ac41c7be5035d4e390a97d 2024-11-24T02:47:16,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/6911aa6184bc4a14a3d7ef25cc67cc57 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/6911aa6184bc4a14a3d7ef25cc67cc57 2024-11-24T02:47:16,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/2cd6682bdfd343c8b05df40b9679a76e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/2cd6682bdfd343c8b05df40b9679a76e 2024-11-24T02:47:16,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/07d6485421bb4c0aa1218a71ed20b8a6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/07d6485421bb4c0aa1218a71ed20b8a6 2024-11-24T02:47:16,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/78e01ffe2b424ca4a7dcde6c4b2cf227 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/78e01ffe2b424ca4a7dcde6c4b2cf227 2024-11-24T02:47:16,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4e05aba722014d5bb2460daebc8aad79 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4e05aba722014d5bb2460daebc8aad79 2024-11-24T02:47:16,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d7431c5e3c8b49b196019d000dd25209 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d7431c5e3c8b49b196019d000dd25209 2024-11-24T02:47:16,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d30a32192c4a41d3bdbd5a801285d0cf to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/d30a32192c4a41d3bdbd5a801285d0cf 2024-11-24T02:47:16,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/302a879c5fe24bd0a9f9bb05a337fd1b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/302a879c5fe24bd0a9f9bb05a337fd1b 2024-11-24T02:47:16,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4b5e1be94b554c5a91419e3e319a868c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/4b5e1be94b554c5a91419e3e319a868c 2024-11-24T02:47:16,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/f79b790bf80b4fa5b889010e97ffb493 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/f79b790bf80b4fa5b889010e97ffb493 2024-11-24T02:47:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/701ad3263d424399b55de89518b06057 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/701ad3263d424399b55de89518b06057 2024-11-24T02:47:16,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/542be654b1f5401ba572f271e83e2d0d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/542be654b1f5401ba572f271e83e2d0d 2024-11-24T02:47:16,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/c1a2ff76e3ac4637b3d8bc37292dc639 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/c1a2ff76e3ac4637b3d8bc37292dc639 2024-11-24T02:47:16,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/548779c92c29484c9d81e110d9e383da to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/548779c92c29484c9d81e110d9e383da 2024-11-24T02:47:16,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/77811302add84ade86313aef32a70805, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f75fc015ed704ebaa5e9a077850522c4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ff3b7a4028a2410c9b0b91f3513099d2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/9a83ee683fa24d20aaec6478294dbc01, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b80a91b84a234fb89568feedc02e744f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c94e7d00a9c3441498db165bc96999f5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/a4b568b6d6db43c9a341f2cc42a73463, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c218ac3328864bdb93f947ed1b61343a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/7382367e78884a88a4abbab84d581490, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/e7817bd3d4cd40898faca682decd66bb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bc5bd56ddf3640a9ae536bc063aa98b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/8dbd65cd7ab84fa3aab134a0bde6c60b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/795398d301264ab2bfe3066f57cd235d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/3e0511106bcc41beb1590659a596f9e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bd856396bcda471186240517cda3fd65, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f978b651825a420885b7cef2f6b027ce, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/64605c3be5714d0ebf6e609d9d21fc34, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/741b06b3ed8c43c7b2968d48c61b004a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c31fabf4531548e9b2b5c67195e938e8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/530db6947b7747a49502d4b8e1f0f97a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/fc1fe8fdbff04917b296c95cd0b7bf06, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/4fb81a39bd14424eb032217f81ddb3f8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/d16d6b02ca384dbdae2db0d3e3f92fae, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/99225b814fd947f8b250c410d0d7703c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f1ecf947f33a4e27854083ff98089611, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/21ec06e68ef847b697c0d4f829314887, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ed48bae5a3f0496f930d2852e9eb5700] to archive 2024-11-24T02:47:16,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:47:16,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/77811302add84ade86313aef32a70805 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/77811302add84ade86313aef32a70805 2024-11-24T02:47:16,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f75fc015ed704ebaa5e9a077850522c4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f75fc015ed704ebaa5e9a077850522c4 2024-11-24T02:47:16,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ff3b7a4028a2410c9b0b91f3513099d2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ff3b7a4028a2410c9b0b91f3513099d2 2024-11-24T02:47:16,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/9a83ee683fa24d20aaec6478294dbc01 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/9a83ee683fa24d20aaec6478294dbc01 2024-11-24T02:47:16,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b80a91b84a234fb89568feedc02e744f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b80a91b84a234fb89568feedc02e744f 2024-11-24T02:47:16,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c94e7d00a9c3441498db165bc96999f5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c94e7d00a9c3441498db165bc96999f5 2024-11-24T02:47:16,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/a4b568b6d6db43c9a341f2cc42a73463 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/a4b568b6d6db43c9a341f2cc42a73463 2024-11-24T02:47:16,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c218ac3328864bdb93f947ed1b61343a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c218ac3328864bdb93f947ed1b61343a 2024-11-24T02:47:16,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/7382367e78884a88a4abbab84d581490 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/7382367e78884a88a4abbab84d581490 2024-11-24T02:47:16,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/e7817bd3d4cd40898faca682decd66bb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/e7817bd3d4cd40898faca682decd66bb 2024-11-24T02:47:16,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bc5bd56ddf3640a9ae536bc063aa98b7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bc5bd56ddf3640a9ae536bc063aa98b7 2024-11-24T02:47:16,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/8dbd65cd7ab84fa3aab134a0bde6c60b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/8dbd65cd7ab84fa3aab134a0bde6c60b 2024-11-24T02:47:16,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/795398d301264ab2bfe3066f57cd235d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/795398d301264ab2bfe3066f57cd235d 2024-11-24T02:47:16,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/3e0511106bcc41beb1590659a596f9e6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/3e0511106bcc41beb1590659a596f9e6 2024-11-24T02:47:16,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bd856396bcda471186240517cda3fd65 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/bd856396bcda471186240517cda3fd65 2024-11-24T02:47:16,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f978b651825a420885b7cef2f6b027ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f978b651825a420885b7cef2f6b027ce 2024-11-24T02:47:16,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/64605c3be5714d0ebf6e609d9d21fc34 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/64605c3be5714d0ebf6e609d9d21fc34 2024-11-24T02:47:16,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/741b06b3ed8c43c7b2968d48c61b004a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/741b06b3ed8c43c7b2968d48c61b004a 2024-11-24T02:47:16,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c31fabf4531548e9b2b5c67195e938e8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/c31fabf4531548e9b2b5c67195e938e8 2024-11-24T02:47:16,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/530db6947b7747a49502d4b8e1f0f97a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/530db6947b7747a49502d4b8e1f0f97a 2024-11-24T02:47:16,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/fc1fe8fdbff04917b296c95cd0b7bf06 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/fc1fe8fdbff04917b296c95cd0b7bf06 2024-11-24T02:47:16,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/4fb81a39bd14424eb032217f81ddb3f8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/4fb81a39bd14424eb032217f81ddb3f8 2024-11-24T02:47:16,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/d16d6b02ca384dbdae2db0d3e3f92fae to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/d16d6b02ca384dbdae2db0d3e3f92fae 2024-11-24T02:47:16,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/99225b814fd947f8b250c410d0d7703c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/99225b814fd947f8b250c410d0d7703c 2024-11-24T02:47:16,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f1ecf947f33a4e27854083ff98089611 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/f1ecf947f33a4e27854083ff98089611 2024-11-24T02:47:16,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/21ec06e68ef847b697c0d4f829314887 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/21ec06e68ef847b697c0d4f829314887 2024-11-24T02:47:16,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ed48bae5a3f0496f930d2852e9eb5700 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/ed48bae5a3f0496f930d2852e9eb5700 2024-11-24T02:47:16,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/84c3559965b0447fb3e6dbeb5be96be9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ce90368dfa784c048639cd2325af3dc3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/516ed0eb64344847baa59eba3663abae, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/162a6559dd754bc09ff96733974f5ee5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/866dcd44edbe4d2a9d2929067e74f760, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/46fab24121604e38b5325bdbb0b694d7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/96769e1291c2481c9136acb2c136d782, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/56519688df1b4fa294bac4cee810e261, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d45fda496bbf4ea69d3a5e34245eb1b0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/2f6bf588fcad407298c8cf2dffc120e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d9af62d07cca45c18700a754b4e3908e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/43dcaf7af9fb47b1bf3a05c9e5fbc473, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/149e7bf11d044fc8a1b3a234be8f331c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/11b16e2213c245dcb85d84617d40ca15, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c4a010eaa54d46b79a81da6fba593a5d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ecffbd5530024d478995b3351ddd1ae1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/26d771cd70b4402aa739612b5e1bf570, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/13069bc9060b49208d711366915887dc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/4774733e4cb04d8093f85e725500dd53, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/66d1769945e340bfb53313eaa0cbfaa9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/07969fa5ae074449826779c962aece93, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/71602a5d684445bd9e1e3e269d60489b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/73a4951d6c6e4a21a24cfa1661f49738, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/204d8427a33741aa91c35481900fe11a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c67379d97f7443a5922abfc3af00c25f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/9c7ff674397e4d168fcda01e30bf7605, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/e89f9d77527c40d4aeaf09d26e7fa39a] to archive 2024-11-24T02:47:16,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:47:16,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/84c3559965b0447fb3e6dbeb5be96be9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/84c3559965b0447fb3e6dbeb5be96be9 2024-11-24T02:47:16,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ce90368dfa784c048639cd2325af3dc3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ce90368dfa784c048639cd2325af3dc3 2024-11-24T02:47:16,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/516ed0eb64344847baa59eba3663abae to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/516ed0eb64344847baa59eba3663abae 2024-11-24T02:47:16,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/162a6559dd754bc09ff96733974f5ee5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/162a6559dd754bc09ff96733974f5ee5 2024-11-24T02:47:16,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/866dcd44edbe4d2a9d2929067e74f760 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/866dcd44edbe4d2a9d2929067e74f760 2024-11-24T02:47:16,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/46fab24121604e38b5325bdbb0b694d7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/46fab24121604e38b5325bdbb0b694d7 2024-11-24T02:47:16,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/96769e1291c2481c9136acb2c136d782 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/96769e1291c2481c9136acb2c136d782 2024-11-24T02:47:16,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/56519688df1b4fa294bac4cee810e261 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/56519688df1b4fa294bac4cee810e261 2024-11-24T02:47:16,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d45fda496bbf4ea69d3a5e34245eb1b0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d45fda496bbf4ea69d3a5e34245eb1b0 2024-11-24T02:47:16,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/2f6bf588fcad407298c8cf2dffc120e6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/2f6bf588fcad407298c8cf2dffc120e6 2024-11-24T02:47:16,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d9af62d07cca45c18700a754b4e3908e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/d9af62d07cca45c18700a754b4e3908e 2024-11-24T02:47:16,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/43dcaf7af9fb47b1bf3a05c9e5fbc473 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/43dcaf7af9fb47b1bf3a05c9e5fbc473 2024-11-24T02:47:16,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/149e7bf11d044fc8a1b3a234be8f331c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/149e7bf11d044fc8a1b3a234be8f331c 2024-11-24T02:47:16,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/11b16e2213c245dcb85d84617d40ca15 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/11b16e2213c245dcb85d84617d40ca15 2024-11-24T02:47:16,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c4a010eaa54d46b79a81da6fba593a5d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c4a010eaa54d46b79a81da6fba593a5d 2024-11-24T02:47:16,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ecffbd5530024d478995b3351ddd1ae1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/ecffbd5530024d478995b3351ddd1ae1 2024-11-24T02:47:16,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/26d771cd70b4402aa739612b5e1bf570 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/26d771cd70b4402aa739612b5e1bf570 2024-11-24T02:47:16,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/13069bc9060b49208d711366915887dc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/13069bc9060b49208d711366915887dc 2024-11-24T02:47:16,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/4774733e4cb04d8093f85e725500dd53 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/4774733e4cb04d8093f85e725500dd53 2024-11-24T02:47:16,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/66d1769945e340bfb53313eaa0cbfaa9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/66d1769945e340bfb53313eaa0cbfaa9 2024-11-24T02:47:16,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/07969fa5ae074449826779c962aece93 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/07969fa5ae074449826779c962aece93 2024-11-24T02:47:16,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/71602a5d684445bd9e1e3e269d60489b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/71602a5d684445bd9e1e3e269d60489b 2024-11-24T02:47:16,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/73a4951d6c6e4a21a24cfa1661f49738 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/73a4951d6c6e4a21a24cfa1661f49738 2024-11-24T02:47:16,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/204d8427a33741aa91c35481900fe11a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/204d8427a33741aa91c35481900fe11a 2024-11-24T02:47:16,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c67379d97f7443a5922abfc3af00c25f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/c67379d97f7443a5922abfc3af00c25f 2024-11-24T02:47:16,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/9c7ff674397e4d168fcda01e30bf7605 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/9c7ff674397e4d168fcda01e30bf7605 2024-11-24T02:47:16,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/e89f9d77527c40d4aeaf09d26e7fa39a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/e89f9d77527c40d4aeaf09d26e7fa39a 2024-11-24T02:47:16,117 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/recovered.edits/420.seqid, newMaxSeqId=420, maxSeqId=4 2024-11-24T02:47:16,118 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. 2024-11-24T02:47:16,118 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 98fb84d8404fffb712280e1a9e01a3ce: Waiting for close lock at 1732416434335Running coprocessor pre-close hooks at 1732416434335Disabling compacts and flushes for region at 1732416434335Disabling writes for close at 1732416434335Obtaining lock to block concurrent updates at 1732416434335Preparing flush snapshotting stores in 98fb84d8404fffb712280e1a9e01a3ce at 1732416434335Finished memstore snapshotting TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732416434336 (+1 ms)Flushing stores of TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce. at 1732416434336Flushing 98fb84d8404fffb712280e1a9e01a3ce/A: creating writer at 1732416434336Flushing mob file 98fb84d8404fffb712280e1a9e01a3ce/A: appending metadata at 1732416434344 (+8 ms)Flushing mob file 98fb84d8404fffb712280e1a9e01a3ce/A: closing flushed file at 1732416434344Flushing 98fb84d8404fffb712280e1a9e01a3ce/A: appending metadata at 1732416434756 (+412 ms)Flushing 98fb84d8404fffb712280e1a9e01a3ce/A: closing flushed file at 1732416434756Flushing 98fb84d8404fffb712280e1a9e01a3ce/B: creating writer at 1732416435167 (+411 ms)Flushing 98fb84d8404fffb712280e1a9e01a3ce/B: appending metadata at 1732416435171 (+4 ms)Flushing 98fb84d8404fffb712280e1a9e01a3ce/B: closing flushed file at 1732416435172 (+1 ms)Flushing 98fb84d8404fffb712280e1a9e01a3ce/C: creating writer at 1732416435582 (+410 ms)Flushing 98fb84d8404fffb712280e1a9e01a3ce/C: appending metadata at 1732416435585 (+3 ms)Flushing 98fb84d8404fffb712280e1a9e01a3ce/C: closing flushed file at 1732416435585Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48f2844e: reopening flushed file at 1732416435996 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64bf7740: reopening flushed file at 1732416436001 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@af26078: reopening flushed file at 1732416436007 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 98fb84d8404fffb712280e1a9e01a3ce in 1678ms, sequenceid=417, compaction requested=true at 1732416436013 (+6 ms)Writing region close event to WAL at 1732416436114 (+101 ms)Running coprocessor post-close hooks at 1732416436118 (+4 ms)Closed at 1732416436118 2024-11-24T02:47:16,120 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=98fb84d8404fffb712280e1a9e01a3ce, regionState=CLOSED 2024-11-24T02:47:16,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:47:16,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-24T02:47:16,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 98fb84d8404fffb712280e1a9e01a3ce, server=7c69a60bd8f6,39461,1732416369827 in 1.9410 sec 2024-11-24T02:47:16,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-24T02:47:16,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=98fb84d8404fffb712280e1a9e01a3ce, UNASSIGN in 1.9470 sec 2024-11-24T02:47:16,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T02:47:16,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9500 sec 2024-11-24T02:47:16,130 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416436130"}]},"ts":"1732416436130"} 2024-11-24T02:47:16,132 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:47:16,132 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:47:16,135 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9620 sec 2024-11-24T02:47:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T02:47:16,317 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:16,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:47:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:16,319 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:16,319 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:47:16,322 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,324 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/recovered.edits] 2024-11-24T02:47:16,327 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/23dd602e8bf84bbe920a66324f6cb9b0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/23dd602e8bf84bbe920a66324f6cb9b0 2024-11-24T02:47:16,328 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/395177e748ac40ab82c00f475ed93246 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/395177e748ac40ab82c00f475ed93246 2024-11-24T02:47:16,330 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/67870c03e4004236a21cd1b6e39d8549 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/A/67870c03e4004236a21cd1b6e39d8549 2024-11-24T02:47:16,333 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/0705bdfdff38430baf8540c82d716281 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/0705bdfdff38430baf8540c82d716281 2024-11-24T02:47:16,335 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/49a5fc4b4a2d4f6a96028215353bde9d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/49a5fc4b4a2d4f6a96028215353bde9d 2024-11-24T02:47:16,337 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b5c0eed7f2b04fc0935c8e5a74d95136 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/B/b5c0eed7f2b04fc0935c8e5a74d95136 2024-11-24T02:47:16,339 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/29ab4fa5f0a44d9cb27f1b33fef55bad to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/29ab4fa5f0a44d9cb27f1b33fef55bad 2024-11-24T02:47:16,341 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/aedf605d7ebf43c2ab31440e12391897 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/aedf605d7ebf43c2ab31440e12391897 2024-11-24T02:47:16,344 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/f335f4b9dd4f4ac28b2d9a684d23ab44 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/C/f335f4b9dd4f4ac28b2d9a684d23ab44 2024-11-24T02:47:16,347 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/recovered.edits/420.seqid to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce/recovered.edits/420.seqid 2024-11-24T02:47:16,348 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,348 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:47:16,348 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:47:16,349 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T02:47:16,354 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124020ee39eb90f4e98ae9f37c861aafedd_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124020ee39eb90f4e98ae9f37c861aafedd_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,355 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112403dea72eba464ade90b75ca90fc77f4d_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112403dea72eba464ade90b75ca90fc77f4d_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,357 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404ece38f86ff4785a7320b3b402b8da9_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404ece38f86ff4785a7320b3b402b8da9_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,358 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112405175ccbf8c2408e84c3d8db4caa2e79_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112405175ccbf8c2408e84c3d8db4caa2e79_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,359 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241844ce9773b14cf5a08b5447d0585e2c_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241844ce9773b14cf5a08b5447d0585e2c_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,360 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e0fc59723ed4d98ba8313cb5eab6ef2_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e0fc59723ed4d98ba8313cb5eab6ef2_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,362 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e2cf4d42b414fbb9e8855b2161e146a_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e2cf4d42b414fbb9e8855b2161e146a_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,363 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124271d682bf0c44092bcea2694d692d557_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124271d682bf0c44092bcea2694d692d557_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,364 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124472781701e0c4178be140d5f374c99e1_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124472781701e0c4178be140d5f374c99e1_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,365 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ebda4bb3ce840faa435266fe5b289d5_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244ebda4bb3ce840faa435266fe5b289d5_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,367 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112450973bcf917d412694a9db37003e87a0_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112450973bcf917d412694a9db37003e87a0_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,368 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124682466c6ab81471a9e3ed655a8f51837_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124682466c6ab81471a9e3ed655a8f51837_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,369 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124778c3c49411b4255ae22b11c66ecc88b_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124778c3c49411b4255ae22b11c66ecc88b_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,370 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2c122d62f9145169695418fdfb760fe_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2c122d62f9145169695418fdfb760fe_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,372 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c6314f755114466c83fa1241eb7437fa_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c6314f755114466c83fa1241eb7437fa_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,373 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d8511e0e6a9f49c09b72dca952f5f0c3_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d8511e0e6a9f49c09b72dca952f5f0c3_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,374 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d9e92c73eaab4b23a076d9e50104f6ee_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d9e92c73eaab4b23a076d9e50104f6ee_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,375 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e5b1b2ad21914f9db58313d229108a44_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e5b1b2ad21914f9db58313d229108a44_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,378 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ec8394e9db8b45f19ab1690c06d400bc_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ec8394e9db8b45f19ab1690c06d400bc_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,379 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f60201caa3f04b4b8ecc9636a1b72528_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f60201caa3f04b4b8ecc9636a1b72528_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,380 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb66b3d79a794c50b453d05bb30e4b16_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fb66b3d79a794c50b453d05bb30e4b16_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,382 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fdb13d1beadd44c1aa3e886e67c0e011_98fb84d8404fffb712280e1a9e01a3ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124fdb13d1beadd44c1aa3e886e67c0e011_98fb84d8404fffb712280e1a9e01a3ce 2024-11-24T02:47:16,382 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:47:16,384 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:16,387 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:47:16,390 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:47:16,392 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:16,392 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:47:16,392 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416436392"}]},"ts":"9223372036854775807"} 2024-11-24T02:47:16,396 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:47:16,396 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 98fb84d8404fffb712280e1a9e01a3ce, NAME => 'TestAcidGuarantees,,1732416400589.98fb84d8404fffb712280e1a9e01a3ce.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:47:16,396 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:47:16,396 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416436396"}]},"ts":"9223372036854775807"} 2024-11-24T02:47:16,399 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:47:16,400 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:16,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-24T02:47:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T02:47:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:47:16,427 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:16,435 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 240), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 957), ProcessCount=11 (was 11), AvailableMemoryMB=10486 (was 10513) 2024-11-24T02:47:16,445 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=696, ProcessCount=11, AvailableMemoryMB=10486 2024-11-24T02:47:16,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:47:16,447 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:47:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:16,449 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:47:16,449 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:16,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-24T02:47:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:47:16,450 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:47:16,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742070_1246 (size=959) 2024-11-24T02:47:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:47:16,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:47:16,858 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:47:16,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742071_1247 (size=53) 2024-11-24T02:47:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:47:17,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:47:17,271 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a9560919f24867d41b2f73bc8d1a4195, disabling compactions & flushes 2024-11-24T02:47:17,272 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:17,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:17,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. after waiting 0 ms 2024-11-24T02:47:17,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:17,272 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:17,272 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a9560919f24867d41b2f73bc8d1a4195: Waiting for close lock at 1732416437271Disabling compacts and flushes for region at 1732416437271Disabling writes for close at 1732416437272 (+1 ms)Writing region close event to WAL at 1732416437272Closed at 1732416437272 2024-11-24T02:47:17,273 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:47:17,273 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416437273"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416437273"}]},"ts":"1732416437273"} 2024-11-24T02:47:17,275 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:47:17,276 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:47:17,276 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416437276"}]},"ts":"1732416437276"} 2024-11-24T02:47:17,277 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:47:17,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, ASSIGN}] 2024-11-24T02:47:17,279 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, ASSIGN 2024-11-24T02:47:17,280 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=false 2024-11-24T02:47:17,431 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a9560919f24867d41b2f73bc8d1a4195, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:17,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, ASSIGN because future has completed 2024-11-24T02:47:17,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:47:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:47:17,589 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:17,590 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:47:17,590 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,590 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:47:17,590 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,591 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,592 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,594 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9560919f24867d41b2f73bc8d1a4195 columnFamilyName A 2024-11-24T02:47:17,595 DEBUG [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:17,595 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] regionserver.HStore(327): Store=a9560919f24867d41b2f73bc8d1a4195/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:17,595 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,597 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9560919f24867d41b2f73bc8d1a4195 columnFamilyName B 2024-11-24T02:47:17,597 DEBUG [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:17,597 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] regionserver.HStore(327): Store=a9560919f24867d41b2f73bc8d1a4195/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:17,597 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,598 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9560919f24867d41b2f73bc8d1a4195 columnFamilyName C 2024-11-24T02:47:17,598 DEBUG [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:17,599 INFO [StoreOpener-a9560919f24867d41b2f73bc8d1a4195-1 {}] regionserver.HStore(327): Store=a9560919f24867d41b2f73bc8d1a4195/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:17,599 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,599 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,600 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,600 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,600 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,601 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:47:17,602 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,603 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:47:17,603 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened a9560919f24867d41b2f73bc8d1a4195; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61781453, jitterRate=-0.07938461005687714}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:47:17,603 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:17,604 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for a9560919f24867d41b2f73bc8d1a4195: Running coprocessor pre-open hook at 1732416437591Writing region info on filesystem at 1732416437591Initializing all the Stores at 1732416437592 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416437592Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416437592Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416437592Cleaning up temporary data from old regions at 1732416437600 (+8 ms)Running coprocessor post-open hooks at 1732416437603 (+3 ms)Region opened successfully at 1732416437604 (+1 ms) 2024-11-24T02:47:17,605 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., pid=61, masterSystemTime=1732416437585 2024-11-24T02:47:17,606 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:17,606 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:17,607 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a9560919f24867d41b2f73bc8d1a4195, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:17,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:47:17,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T02:47:17,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 in 176 msec 2024-11-24T02:47:17,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-24T02:47:17,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, ASSIGN in 332 msec 2024-11-24T02:47:17,613 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:47:17,613 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416437613"}]},"ts":"1732416437613"} 2024-11-24T02:47:17,614 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:47:17,615 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:47:17,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-24T02:47:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-24T02:47:18,597 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:18,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,600 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@752e07c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,605 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:47:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,43749,1732416369043 2024-11-24T02:47:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e52840c 2024-11-24T02:47:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:47:18,607 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:47:18,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,609 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f8ab9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,610 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5539dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,612 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58dec3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,613 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57fc3b7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,615 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c307aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f65e74d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,618 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5018a696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,619 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402b8ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,622 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@718fc983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,623 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78973ec3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,625 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18283aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,627 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d765118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,628 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8c19c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,630 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20aa5a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,632 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d604b28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,633 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa07b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,636 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143a3089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8922c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:18,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:18,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6710ba91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:18,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37dd5517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:18,644 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,644 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,644 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,647 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,648 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,649 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,649 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,650 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,651 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,652 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-24T02:47:18,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,654 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:18,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:47:18,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,660 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:18,661 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:18,661 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:18,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:18,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:47:18,665 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:18,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:18,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:18,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/96a0652f9f0f42de93f63fa40582a127 is 50, key is test_row_0/A:col10/1732416438660/Put/seqid=0 2024-11-24T02:47:18,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742072_1248 (size=16681) 2024-11-24T02:47:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416498720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416498720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416498721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416498725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416498725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:47:18,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:47:18,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:18,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:18,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:18,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:18,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416498837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416498837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416498837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416498837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416498837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:18,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:18,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:47:18,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:47:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:18,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416499047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416499047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416499047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416499047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416499047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/96a0652f9f0f42de93f63fa40582a127 2024-11-24T02:47:19,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:47:19,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:19,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:19,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:19,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:19,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:19,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/588c9c5654b2471785762adce2d710ea is 50, key is test_row_0/B:col10/1732416438660/Put/seqid=0 2024-11-24T02:47:19,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742073_1249 (size=12001) 2024-11-24T02:47:19,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/588c9c5654b2471785762adce2d710ea 2024-11-24T02:47:19,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/8387e0c7007c4c99bcd417e31c7585bf is 50, key is test_row_0/C:col10/1732416438660/Put/seqid=0 2024-11-24T02:47:19,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742074_1250 (size=12001) 2024-11-24T02:47:19,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/8387e0c7007c4c99bcd417e31c7585bf 2024-11-24T02:47:19,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/96a0652f9f0f42de93f63fa40582a127 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/96a0652f9f0f42de93f63fa40582a127 2024-11-24T02:47:19,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/96a0652f9f0f42de93f63fa40582a127, entries=250, sequenceid=12, filesize=16.3 K 2024-11-24T02:47:19,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/588c9c5654b2471785762adce2d710ea as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/588c9c5654b2471785762adce2d710ea 2024-11-24T02:47:19,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/588c9c5654b2471785762adce2d710ea, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T02:47:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/8387e0c7007c4c99bcd417e31c7585bf as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8387e0c7007c4c99bcd417e31c7585bf 2024-11-24T02:47:19,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8387e0c7007c4c99bcd417e31c7585bf, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T02:47:19,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9560919f24867d41b2f73bc8d1a4195 in 540ms, sequenceid=12, compaction requested=false 2024-11-24T02:47:19,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:19,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-24T02:47:19,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:47:19,277 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:47:19,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ad1384ecc64c40cc907a0c135da30604 is 50, key is test_row_0/A:col10/1732416438719/Put/seqid=0 2024-11-24T02:47:19,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742075_1251 (size=12001) 2024-11-24T02:47:19,338 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ad1384ecc64c40cc907a0c135da30604 2024-11-24T02:47:19,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:19,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/db2291e5a6434a55a104d01c8e5e74ee is 50, key is test_row_0/B:col10/1732416438719/Put/seqid=0 2024-11-24T02:47:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416499363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416499363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416499364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416499365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416499364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742076_1252 (size=12001) 2024-11-24T02:47:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416499477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416499478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416499478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416499478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416499479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416499687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416499687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416499687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416499687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416499687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:47:19,795 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/db2291e5a6434a55a104d01c8e5e74ee 2024-11-24T02:47:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/933c974d88484e789ad6119aacb1cd09 is 50, key is test_row_0/C:col10/1732416438719/Put/seqid=0 2024-11-24T02:47:19,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742077_1253 (size=12001) 2024-11-24T02:47:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416499997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416499997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:19,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416499997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416499997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416499997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/933c974d88484e789ad6119aacb1cd09 2024-11-24T02:47:20,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ad1384ecc64c40cc907a0c135da30604 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ad1384ecc64c40cc907a0c135da30604 2024-11-24T02:47:20,245 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ad1384ecc64c40cc907a0c135da30604, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:47:20,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/db2291e5a6434a55a104d01c8e5e74ee as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/db2291e5a6434a55a104d01c8e5e74ee 2024-11-24T02:47:20,253 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/db2291e5a6434a55a104d01c8e5e74ee, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:47:20,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/933c974d88484e789ad6119aacb1cd09 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/933c974d88484e789ad6119aacb1cd09 2024-11-24T02:47:20,263 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/933c974d88484e789ad6119aacb1cd09, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:47:20,267 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9560919f24867d41b2f73bc8d1a4195 in 989ms, sequenceid=37, compaction requested=false 2024-11-24T02:47:20,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:20,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:20,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-24T02:47:20,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-24T02:47:20,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T02:47:20,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6080 sec 2024-11-24T02:47:20,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.6220 sec 2024-11-24T02:47:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:20,512 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:20,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:47:20,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/01b98497e5e247f2803e2ec91e49e7ca is 50, key is test_row_0/A:col10/1732416440510/Put/seqid=0 2024-11-24T02:47:20,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416500545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416500546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416500548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416500554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416500557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742078_1254 (size=14341) 2024-11-24T02:47:20,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416500656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416500656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416500667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416500667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416500667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T02:47:20,797 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:20,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-24T02:47:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:47:20,800 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:20,801 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:20,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416500866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416500866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416500876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416500877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416500877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:20,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:47:20,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:20,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:20,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:20,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:20,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:20,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:20,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/01b98497e5e247f2803e2ec91e49e7ca 2024-11-24T02:47:20,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/251af6a86472494ebdf4e11a572c5f9d is 50, key is test_row_0/B:col10/1732416440510/Put/seqid=0 2024-11-24T02:47:21,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742079_1255 (size=12001) 2024-11-24T02:47:21,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/251af6a86472494ebdf4e11a572c5f9d 2024-11-24T02:47:21,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/07fca0e6133446e2aafc948e15022bce is 50, key is test_row_0/C:col10/1732416440510/Put/seqid=0 2024-11-24T02:47:21,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742080_1256 (size=12001) 2024-11-24T02:47:21,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/07fca0e6133446e2aafc948e15022bce 2024-11-24T02:47:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:47:21,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/01b98497e5e247f2803e2ec91e49e7ca as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/01b98497e5e247f2803e2ec91e49e7ca 2024-11-24T02:47:21,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/01b98497e5e247f2803e2ec91e49e7ca, entries=200, sequenceid=52, filesize=14.0 K 2024-11-24T02:47:21,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/251af6a86472494ebdf4e11a572c5f9d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/251af6a86472494ebdf4e11a572c5f9d 2024-11-24T02:47:21,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/251af6a86472494ebdf4e11a572c5f9d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T02:47:21,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/07fca0e6133446e2aafc948e15022bce as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/07fca0e6133446e2aafc948e15022bce 2024-11-24T02:47:21,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/07fca0e6133446e2aafc948e15022bce, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T02:47:21,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for a9560919f24867d41b2f73bc8d1a4195 in 635ms, sequenceid=52, compaction requested=true 2024-11-24T02:47:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:21,148 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:21,148 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:21,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:21,150 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:21,150 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:21,150 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:21,151 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:21,151 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,151 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,151 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/96a0652f9f0f42de93f63fa40582a127, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ad1384ecc64c40cc907a0c135da30604, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/01b98497e5e247f2803e2ec91e49e7ca] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=42.0 K 2024-11-24T02:47:21,151 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/588c9c5654b2471785762adce2d710ea, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/db2291e5a6434a55a104d01c8e5e74ee, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/251af6a86472494ebdf4e11a572c5f9d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.2 K 2024-11-24T02:47:21,151 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a0652f9f0f42de93f63fa40582a127, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732416438658 2024-11-24T02:47:21,151 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 588c9c5654b2471785762adce2d710ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732416438660 2024-11-24T02:47:21,152 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting db2291e5a6434a55a104d01c8e5e74ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416438719 2024-11-24T02:47:21,152 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad1384ecc64c40cc907a0c135da30604, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416438719 2024-11-24T02:47:21,152 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 251af6a86472494ebdf4e11a572c5f9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416439363 2024-11-24T02:47:21,152 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01b98497e5e247f2803e2ec91e49e7ca, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416439363 2024-11-24T02:47:21,167 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:21,167 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/252b9f891be24bc9977b2a8207dc9bb9 is 50, key is test_row_0/B:col10/1732416440510/Put/seqid=0 2024-11-24T02:47:21,179 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:21,179 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/aebae64ba45745379863affecb0b504b is 50, key is test_row_0/A:col10/1732416440510/Put/seqid=0 2024-11-24T02:47:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:21,185 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:21,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:47:21,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416501200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416501202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416501203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742081_1257 (size=12104) 2024-11-24T02:47:21,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416501206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416501206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,223 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/252b9f891be24bc9977b2a8207dc9bb9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/252b9f891be24bc9977b2a8207dc9bb9 2024-11-24T02:47:21,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/2a38fd17f0d044c98823def17d821ed6 is 50, key is test_row_0/A:col10/1732416440552/Put/seqid=0 2024-11-24T02:47:21,246 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into 252b9f891be24bc9977b2a8207dc9bb9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:21,246 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:21,246 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=13, startTime=1732416441148; duration=0sec 2024-11-24T02:47:21,246 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:21,246 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:21,246 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:21,247 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:21,247 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:21,248 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,248 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8387e0c7007c4c99bcd417e31c7585bf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/933c974d88484e789ad6119aacb1cd09, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/07fca0e6133446e2aafc948e15022bce] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.2 K 2024-11-24T02:47:21,248 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8387e0c7007c4c99bcd417e31c7585bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732416438660 2024-11-24T02:47:21,248 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 933c974d88484e789ad6119aacb1cd09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416438719 2024-11-24T02:47:21,249 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 07fca0e6133446e2aafc948e15022bce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416439363 2024-11-24T02:47:21,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742082_1258 (size=12104) 2024-11-24T02:47:21,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,272 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/aebae64ba45745379863affecb0b504b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/aebae64ba45745379863affecb0b504b 2024-11-24T02:47:21,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742083_1259 (size=12001) 2024-11-24T02:47:21,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/2a38fd17f0d044c98823def17d821ed6 2024-11-24T02:47:21,287 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into aebae64ba45745379863affecb0b504b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:21,287 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:21,288 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=13, startTime=1732416441148; duration=0sec 2024-11-24T02:47:21,288 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:21,288 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:21,292 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:21,293 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/05c7d653566749b5b2d9d9e781f6096b is 50, key is test_row_0/C:col10/1732416440510/Put/seqid=0 2024-11-24T02:47:21,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/fa55d8720e1d43b3a2d50c1ff0f72077 is 50, key is test_row_0/B:col10/1732416440552/Put/seqid=0 2024-11-24T02:47:21,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416501317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416501317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416501317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416501327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416501327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742084_1260 (size=12104) 2024-11-24T02:47:21,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742085_1261 (size=12001) 2024-11-24T02:47:21,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/fa55d8720e1d43b3a2d50c1ff0f72077 2024-11-24T02:47:21,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/4a27ba4fb2a74ad9b7c9999d80977463 is 50, key is test_row_0/C:col10/1732416440552/Put/seqid=0 2024-11-24T02:47:21,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:21,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:21,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:47:21,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742086_1262 (size=12001) 2024-11-24T02:47:21,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/4a27ba4fb2a74ad9b7c9999d80977463 2024-11-24T02:47:21,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/2a38fd17f0d044c98823def17d821ed6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a38fd17f0d044c98823def17d821ed6 2024-11-24T02:47:21,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a38fd17f0d044c98823def17d821ed6, entries=150, sequenceid=76, filesize=11.7 K 2024-11-24T02:47:21,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/fa55d8720e1d43b3a2d50c1ff0f72077 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/fa55d8720e1d43b3a2d50c1ff0f72077 2024-11-24T02:47:21,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/fa55d8720e1d43b3a2d50c1ff0f72077, entries=150, sequenceid=76, filesize=11.7 K 2024-11-24T02:47:21,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/4a27ba4fb2a74ad9b7c9999d80977463 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4a27ba4fb2a74ad9b7c9999d80977463 2024-11-24T02:47:21,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4a27ba4fb2a74ad9b7c9999d80977463, entries=150, sequenceid=76, filesize=11.7 K 2024-11-24T02:47:21,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a9560919f24867d41b2f73bc8d1a4195 in 286ms, sequenceid=76, compaction requested=false 2024-11-24T02:47:21,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:21,531 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:21,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:47:21,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ee276d4e6c984d3f8fd4f666b9ef228d is 50, key is test_row_0/A:col10/1732416441203/Put/seqid=0 2024-11-24T02:47:21,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:21,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416501574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416501574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416501576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416501577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416501578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742087_1263 (size=12001) 2024-11-24T02:47:21,682 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:47:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416501697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416501698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416501698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416501698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416501698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,744 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/05c7d653566749b5b2d9d9e781f6096b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/05c7d653566749b5b2d9d9e781f6096b 2024-11-24T02:47:21,750 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into 05c7d653566749b5b2d9d9e781f6096b(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:21,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:21,750 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=13, startTime=1732416441148; duration=0sec 2024-11-24T02:47:21,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:21,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:21,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:21,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:21,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:21,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416501917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416501917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416501917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416501917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416501918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:47:22,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ee276d4e6c984d3f8fd4f666b9ef228d 2024-11-24T02:47:22,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:22,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:22,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:22,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:22,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/16194f941c5c4afa8c46af77e9807577 is 50, key is test_row_0/B:col10/1732416441203/Put/seqid=0 2024-11-24T02:47:22,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742088_1264 (size=12001) 2024-11-24T02:47:22,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/16194f941c5c4afa8c46af77e9807577 2024-11-24T02:47:22,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/3e2b68c79259496b8b5d5d741214c4b2 is 50, key is test_row_0/C:col10/1732416441203/Put/seqid=0 2024-11-24T02:47:22,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742089_1265 (size=12001) 2024-11-24T02:47:22,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/3e2b68c79259496b8b5d5d741214c4b2 2024-11-24T02:47:22,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ee276d4e6c984d3f8fd4f666b9ef228d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ee276d4e6c984d3f8fd4f666b9ef228d 2024-11-24T02:47:22,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ee276d4e6c984d3f8fd4f666b9ef228d, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T02:47:22,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/16194f941c5c4afa8c46af77e9807577 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/16194f941c5c4afa8c46af77e9807577 2024-11-24T02:47:22,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/16194f941c5c4afa8c46af77e9807577, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T02:47:22,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/3e2b68c79259496b8b5d5d741214c4b2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3e2b68c79259496b8b5d5d741214c4b2 2024-11-24T02:47:22,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3e2b68c79259496b8b5d5d741214c4b2, entries=150, sequenceid=94, filesize=11.7 K 2024-11-24T02:47:22,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a9560919f24867d41b2f73bc8d1a4195 in 609ms, sequenceid=94, compaction requested=true 2024-11-24T02:47:22,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:22,140 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:22,142 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:22,142 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:22,142 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:22,142 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/aebae64ba45745379863affecb0b504b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a38fd17f0d044c98823def17d821ed6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ee276d4e6c984d3f8fd4f666b9ef228d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.3 K 2024-11-24T02:47:22,143 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting aebae64ba45745379863affecb0b504b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416439363 2024-11-24T02:47:22,144 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a38fd17f0d044c98823def17d821ed6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732416440541 2024-11-24T02:47:22,144 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee276d4e6c984d3f8fd4f666b9ef228d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416441203 2024-11-24T02:47:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:22,148 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:22,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:22,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:22,150 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:22,150 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:22,150 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:22,150 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/252b9f891be24bc9977b2a8207dc9bb9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/fa55d8720e1d43b3a2d50c1ff0f72077, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/16194f941c5c4afa8c46af77e9807577] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.3 K 2024-11-24T02:47:22,152 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 252b9f891be24bc9977b2a8207dc9bb9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416439363 2024-11-24T02:47:22,152 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting fa55d8720e1d43b3a2d50c1ff0f72077, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732416440541 2024-11-24T02:47:22,152 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 16194f941c5c4afa8c46af77e9807577, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416441203 2024-11-24T02:47:22,153 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:22,154 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/4fdd5f0f1fa14a50bd7ce099cac7c93f is 50, key is test_row_0/A:col10/1732416441203/Put/seqid=0 2024-11-24T02:47:22,166 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:22,167 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2e739069a9634887af9fea70714c5bd9 is 50, key is test_row_0/B:col10/1732416441203/Put/seqid=0 2024-11-24T02:47:22,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742090_1266 (size=12207) 2024-11-24T02:47:22,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-24T02:47:22,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:22,195 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T02:47:22,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/6a01f50e965e4147b308b2fe8868a37c is 50, key is test_row_0/A:col10/1732416441576/Put/seqid=0 2024-11-24T02:47:22,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742091_1267 (size=12207) 2024-11-24T02:47:22,213 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/4fdd5f0f1fa14a50bd7ce099cac7c93f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/4fdd5f0f1fa14a50bd7ce099cac7c93f 2024-11-24T02:47:22,226 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into 4fdd5f0f1fa14a50bd7ce099cac7c93f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:22,226 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:22,226 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=13, startTime=1732416442140; duration=0sec 2024-11-24T02:47:22,226 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:22,226 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:22,226 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:22,228 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:22,228 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:22,229 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:22,230 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/05c7d653566749b5b2d9d9e781f6096b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4a27ba4fb2a74ad9b7c9999d80977463, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3e2b68c79259496b8b5d5d741214c4b2] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.3 K 2024-11-24T02:47:22,230 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c7d653566749b5b2d9d9e781f6096b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416439363 2024-11-24T02:47:22,231 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a27ba4fb2a74ad9b7c9999d80977463, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732416440541 2024-11-24T02:47:22,231 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2b68c79259496b8b5d5d741214c4b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416441203 2024-11-24T02:47:22,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:22,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416502246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416502248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742092_1268 (size=12001) 2024-11-24T02:47:22,257 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/6a01f50e965e4147b308b2fe8868a37c 2024-11-24T02:47:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416502253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416502253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416502254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,274 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#221 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:22,274 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/b7fdc8d77ba942f7b1a35a6244783600 is 50, key is test_row_0/C:col10/1732416441203/Put/seqid=0 2024-11-24T02:47:22,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/99aad9b4fb04425fa425c44ae238f672 is 50, key is test_row_0/B:col10/1732416441576/Put/seqid=0 2024-11-24T02:47:22,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742093_1269 (size=12207) 2024-11-24T02:47:22,297 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/b7fdc8d77ba942f7b1a35a6244783600 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b7fdc8d77ba942f7b1a35a6244783600 2024-11-24T02:47:22,302 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into b7fdc8d77ba942f7b1a35a6244783600(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:22,302 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:22,302 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=13, startTime=1732416442148; duration=0sec 2024-11-24T02:47:22,302 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:22,302 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:22,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742094_1270 (size=12001) 2024-11-24T02:47:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416502356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416502356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416502367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416502367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416502367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416502566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416502566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416502577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416502577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416502577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,621 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2e739069a9634887af9fea70714c5bd9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2e739069a9634887af9fea70714c5bd9 2024-11-24T02:47:22,628 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into 2e739069a9634887af9fea70714c5bd9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:22,628 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:22,628 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=13, startTime=1732416442148; duration=0sec 2024-11-24T02:47:22,628 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:22,628 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:22,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/99aad9b4fb04425fa425c44ae238f672 2024-11-24T02:47:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0959399b436f42e5a29a7392a41cf891 is 50, key is test_row_0/C:col10/1732416441576/Put/seqid=0 2024-11-24T02:47:22,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742095_1271 (size=12001) 2024-11-24T02:47:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416502876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416502878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416502887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416502887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416502887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:22,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:22,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:47:23,117 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0959399b436f42e5a29a7392a41cf891 2024-11-24T02:47:23,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/6a01f50e965e4147b308b2fe8868a37c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/6a01f50e965e4147b308b2fe8868a37c 2024-11-24T02:47:23,135 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/6a01f50e965e4147b308b2fe8868a37c, entries=150, sequenceid=115, filesize=11.7 K 2024-11-24T02:47:23,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/99aad9b4fb04425fa425c44ae238f672 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/99aad9b4fb04425fa425c44ae238f672 2024-11-24T02:47:23,143 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/99aad9b4fb04425fa425c44ae238f672, entries=150, sequenceid=115, filesize=11.7 K 2024-11-24T02:47:23,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0959399b436f42e5a29a7392a41cf891 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0959399b436f42e5a29a7392a41cf891 2024-11-24T02:47:23,152 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0959399b436f42e5a29a7392a41cf891, entries=150, sequenceid=115, filesize=11.7 K 2024-11-24T02:47:23,155 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for a9560919f24867d41b2f73bc8d1a4195 in 960ms, sequenceid=115, compaction requested=false 2024-11-24T02:47:23,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:23,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:23,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-24T02:47:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-24T02:47:23,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-24T02:47:23,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3560 sec 2024-11-24T02:47:23,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.3620 sec 2024-11-24T02:47:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:23,389 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:23,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-24T02:47:23,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f89eda28c7aa43d787f7ad45a3c2aabe is 50, key is test_row_0/A:col10/1732416443389/Put/seqid=0 2024-11-24T02:47:23,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742096_1272 (size=14541) 2024-11-24T02:47:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416503444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416503444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416503445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416503451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416503451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416503557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416503557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416503557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416503567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416503567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416503766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416503766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416503767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416503777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416503777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:23,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f89eda28c7aa43d787f7ad45a3c2aabe 2024-11-24T02:47:23,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b1cf274adbe2479cb7584d26c42b8116 is 50, key is test_row_0/B:col10/1732416443389/Put/seqid=0 2024-11-24T02:47:23,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742097_1273 (size=12151) 2024-11-24T02:47:24,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416504076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416504076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416504076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416504086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416504086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b1cf274adbe2479cb7584d26c42b8116 2024-11-24T02:47:24,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/8e3097ba7d1e479bbe882f821ef5d832 is 50, key is test_row_0/C:col10/1732416443389/Put/seqid=0 2024-11-24T02:47:24,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742098_1274 (size=12151) 2024-11-24T02:47:24,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416504586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416504586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416504586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416504597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416504597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:24,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:24,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/8e3097ba7d1e479bbe882f821ef5d832 2024-11-24T02:47:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f89eda28c7aa43d787f7ad45a3c2aabe as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f89eda28c7aa43d787f7ad45a3c2aabe 2024-11-24T02:47:24,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f89eda28c7aa43d787f7ad45a3c2aabe, entries=200, sequenceid=137, filesize=14.2 K 2024-11-24T02:47:24,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b1cf274adbe2479cb7584d26c42b8116 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b1cf274adbe2479cb7584d26c42b8116 2024-11-24T02:47:24,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b1cf274adbe2479cb7584d26c42b8116, entries=150, sequenceid=137, filesize=11.9 K 2024-11-24T02:47:24,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/8e3097ba7d1e479bbe882f821ef5d832 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8e3097ba7d1e479bbe882f821ef5d832 2024-11-24T02:47:24,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8e3097ba7d1e479bbe882f821ef5d832, entries=150, sequenceid=137, filesize=11.9 K 2024-11-24T02:47:24,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for a9560919f24867d41b2f73bc8d1a4195 in 1299ms, sequenceid=137, compaction requested=true 2024-11-24T02:47:24,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:24,689 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:24,689 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:24,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:24,690 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:24,690 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:24,690 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:24,690 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2e739069a9634887af9fea70714c5bd9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/99aad9b4fb04425fa425c44ae238f672, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b1cf274adbe2479cb7584d26c42b8116] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.5 K 2024-11-24T02:47:24,691 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:24,691 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:24,691 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:24,691 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/4fdd5f0f1fa14a50bd7ce099cac7c93f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/6a01f50e965e4147b308b2fe8868a37c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f89eda28c7aa43d787f7ad45a3c2aabe] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=37.8 K 2024-11-24T02:47:24,691 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e739069a9634887af9fea70714c5bd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416441203 2024-11-24T02:47:24,692 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fdd5f0f1fa14a50bd7ce099cac7c93f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416441203 2024-11-24T02:47:24,692 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 99aad9b4fb04425fa425c44ae238f672, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732416441564 2024-11-24T02:47:24,692 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a01f50e965e4147b308b2fe8868a37c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732416441564 2024-11-24T02:47:24,692 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b1cf274adbe2479cb7584d26c42b8116, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732416442251 2024-11-24T02:47:24,693 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f89eda28c7aa43d787f7ad45a3c2aabe, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732416442251 2024-11-24T02:47:24,716 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:24,716 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/255549aedf3f4dff9a5712d052585a4d is 50, key is test_row_0/B:col10/1732416443389/Put/seqid=0 2024-11-24T02:47:24,722 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:24,722 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/533152c8e50245558f6791d11bfd8707 is 50, key is test_row_0/A:col10/1732416443389/Put/seqid=0 2024-11-24T02:47:24,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742100_1276 (size=12459) 2024-11-24T02:47:24,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742099_1275 (size=12459) 2024-11-24T02:47:24,736 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/255549aedf3f4dff9a5712d052585a4d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/255549aedf3f4dff9a5712d052585a4d 2024-11-24T02:47:24,739 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/533152c8e50245558f6791d11bfd8707 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/533152c8e50245558f6791d11bfd8707 2024-11-24T02:47:24,743 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into 255549aedf3f4dff9a5712d052585a4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:24,743 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:24,744 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=13, startTime=1732416444689; duration=0sec 2024-11-24T02:47:24,744 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:24,744 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:24,744 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:24,746 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:24,746 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:24,747 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:24,747 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b7fdc8d77ba942f7b1a35a6244783600, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0959399b436f42e5a29a7392a41cf891, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8e3097ba7d1e479bbe882f821ef5d832] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.5 K 2024-11-24T02:47:24,747 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b7fdc8d77ba942f7b1a35a6244783600, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732416441203 2024-11-24T02:47:24,748 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0959399b436f42e5a29a7392a41cf891, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732416441564 2024-11-24T02:47:24,748 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3097ba7d1e479bbe882f821ef5d832, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732416442251 2024-11-24T02:47:24,756 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into 533152c8e50245558f6791d11bfd8707(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:24,756 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:24,756 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=13, startTime=1732416444688; duration=0sec 2024-11-24T02:47:24,756 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:24,757 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:24,771 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:24,772 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/6763711416b94fd087e10cedf60fd346 is 50, key is test_row_0/C:col10/1732416443389/Put/seqid=0 2024-11-24T02:47:24,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742101_1277 (size=12459) 2024-11-24T02:47:24,825 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/6763711416b94fd087e10cedf60fd346 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6763711416b94fd087e10cedf60fd346 2024-11-24T02:47:24,833 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into 6763711416b94fd087e10cedf60fd346(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:24,833 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:24,833 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=13, startTime=1732416444689; duration=0sec 2024-11-24T02:47:24,835 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:24,835 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-24T02:47:24,977 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:24,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-24T02:47:24,981 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:47:24,982 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:24,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:47:25,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-24T02:47:25,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,136 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-24T02:47:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/af4880159d4f4812a057b056385c1b2c is 50, key is test_row_0/A:col10/1732416443450/Put/seqid=0 2024-11-24T02:47:25,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742102_1278 (size=12151) 2024-11-24T02:47:25,154 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/af4880159d4f4812a057b056385c1b2c 2024-11-24T02:47:25,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/6b7c51ef376a464c819063b86182b779 is 50, key is test_row_0/B:col10/1732416443450/Put/seqid=0 2024-11-24T02:47:25,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742103_1279 (size=12151) 2024-11-24T02:47:25,181 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/6b7c51ef376a464c819063b86182b779 2024-11-24T02:47:25,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/d77da9827b614ba4b71c657a80636439 is 50, key is test_row_0/C:col10/1732416443450/Put/seqid=0 2024-11-24T02:47:25,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742104_1280 (size=12151) 2024-11-24T02:47:25,206 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/d77da9827b614ba4b71c657a80636439 2024-11-24T02:47:25,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/af4880159d4f4812a057b056385c1b2c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/af4880159d4f4812a057b056385c1b2c 2024-11-24T02:47:25,215 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/af4880159d4f4812a057b056385c1b2c, entries=150, sequenceid=158, filesize=11.9 K 2024-11-24T02:47:25,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/6b7c51ef376a464c819063b86182b779 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/6b7c51ef376a464c819063b86182b779 2024-11-24T02:47:25,221 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/6b7c51ef376a464c819063b86182b779, entries=150, sequenceid=158, filesize=11.9 K 2024-11-24T02:47:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/d77da9827b614ba4b71c657a80636439 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/d77da9827b614ba4b71c657a80636439 2024-11-24T02:47:25,237 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/d77da9827b614ba4b71c657a80636439, entries=150, sequenceid=158, filesize=11.9 K 2024-11-24T02:47:25,238 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for a9560919f24867d41b2f73bc8d1a4195 in 102ms, sequenceid=158, compaction requested=false 2024-11-24T02:47:25,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:25,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-24T02:47:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-24T02:47:25,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-24T02:47:25,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 258 msec 2024-11-24T02:47:25,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 265 msec 2024-11-24T02:47:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T02:47:25,297 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:25,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-24T02:47:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:47:25,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:25,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:25,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:25,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:47:25,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-24T02:47:25,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:25,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-24T02:47:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-24T02:47:25,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T02:47:25,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-24T02:47:25,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 168 msec 2024-11-24T02:47:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-24T02:47:25,617 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:25,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-24T02:47:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:47:25,620 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:25,621 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:25,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:25,659 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:25,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:47:25,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/2a2fb76d862e45f0b47a7807751be3a0 is 50, key is test_row_0/A:col10/1732416445625/Put/seqid=0 2024-11-24T02:47:25,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742105_1281 (size=12151) 2024-11-24T02:47:25,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416505706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/2a2fb76d862e45f0b47a7807751be3a0 2024-11-24T02:47:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416505709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416505710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416505711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416505712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:47:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b45e5777ef284ce6ae9347a3b1861fd0 is 50, key is test_row_0/B:col10/1732416445625/Put/seqid=0 2024-11-24T02:47:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742106_1282 (size=12151) 2024-11-24T02:47:25,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:47:25,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:25,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:25,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:25,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416505827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416505828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416505827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416505837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416505838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:25,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:25,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:47:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:25,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:25,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:47:26,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416506037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416506037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416506046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416506046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416506046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:47:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b45e5777ef284ce6ae9347a3b1861fd0 2024-11-24T02:47:26,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/eb2093c383e84c49af7211b0ca9bf050 is 50, key is test_row_0/C:col10/1732416445625/Put/seqid=0 2024-11-24T02:47:26,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742107_1283 (size=12151) 2024-11-24T02:47:26,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:47:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:47:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416506347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416506347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416506357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416506357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416506357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:47:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:47:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:26,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/eb2093c383e84c49af7211b0ca9bf050 2024-11-24T02:47:26,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/2a2fb76d862e45f0b47a7807751be3a0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a2fb76d862e45f0b47a7807751be3a0 2024-11-24T02:47:26,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a2fb76d862e45f0b47a7807751be3a0, entries=150, sequenceid=171, filesize=11.9 K 2024-11-24T02:47:26,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b45e5777ef284ce6ae9347a3b1861fd0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b45e5777ef284ce6ae9347a3b1861fd0 2024-11-24T02:47:26,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b45e5777ef284ce6ae9347a3b1861fd0, entries=150, sequenceid=171, filesize=11.9 K 2024-11-24T02:47:26,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/eb2093c383e84c49af7211b0ca9bf050 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/eb2093c383e84c49af7211b0ca9bf050 2024-11-24T02:47:26,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/eb2093c383e84c49af7211b0ca9bf050, entries=150, sequenceid=171, filesize=11.9 K 2024-11-24T02:47:26,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for a9560919f24867d41b2f73bc8d1a4195 in 911ms, sequenceid=171, compaction requested=true 2024-11-24T02:47:26,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:26,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:26,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:26,570 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:26,571 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:26,571 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,571 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,571 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/255549aedf3f4dff9a5712d052585a4d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/6b7c51ef376a464c819063b86182b779, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b45e5777ef284ce6ae9347a3b1861fd0] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.9 K 2024-11-24T02:47:26,571 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/533152c8e50245558f6791d11bfd8707, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/af4880159d4f4812a057b056385c1b2c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a2fb76d862e45f0b47a7807751be3a0] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.9 K 2024-11-24T02:47:26,572 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 255549aedf3f4dff9a5712d052585a4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732416442251 2024-11-24T02:47:26,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 533152c8e50245558f6791d11bfd8707, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732416442251 2024-11-24T02:47:26,572 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b7c51ef376a464c819063b86182b779, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732416443407 2024-11-24T02:47:26,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting af4880159d4f4812a057b056385c1b2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732416443407 2024-11-24T02:47:26,572 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b45e5777ef284ce6ae9347a3b1861fd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416445613 2024-11-24T02:47:26,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2fb76d862e45f0b47a7807751be3a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416445613 2024-11-24T02:47:26,580 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:26,581 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/1933a1be6aa24d19bb6d59200152fa99 is 50, key is test_row_0/B:col10/1732416445625/Put/seqid=0 2024-11-24T02:47:26,591 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:26,592 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/18e8350a4c204bb3bbf6d8c9a1c227e0 is 50, key is test_row_0/A:col10/1732416445625/Put/seqid=0 2024-11-24T02:47:26,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742108_1284 (size=12561) 2024-11-24T02:47:26,601 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/1933a1be6aa24d19bb6d59200152fa99 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/1933a1be6aa24d19bb6d59200152fa99 2024-11-24T02:47:26,607 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into 1933a1be6aa24d19bb6d59200152fa99(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:26,607 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:26,608 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=13, startTime=1732416446570; duration=0sec 2024-11-24T02:47:26,608 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:26,608 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:26,608 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:26,609 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:26,609 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:26,609 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,610 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6763711416b94fd087e10cedf60fd346, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/d77da9827b614ba4b71c657a80636439, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/eb2093c383e84c49af7211b0ca9bf050] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=35.9 K 2024-11-24T02:47:26,610 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6763711416b94fd087e10cedf60fd346, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732416442251 2024-11-24T02:47:26,610 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d77da9827b614ba4b71c657a80636439, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732416443407 2024-11-24T02:47:26,611 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting eb2093c383e84c49af7211b0ca9bf050, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416445613 2024-11-24T02:47:26,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742109_1285 (size=12561) 2024-11-24T02:47:26,627 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:26,627 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/6b3ac041a51b4a718c33c9feffae6241 is 50, key is test_row_0/C:col10/1732416445625/Put/seqid=0 2024-11-24T02:47:26,629 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/18e8350a4c204bb3bbf6d8c9a1c227e0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/18e8350a4c204bb3bbf6d8c9a1c227e0 2024-11-24T02:47:26,635 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into 18e8350a4c204bb3bbf6d8c9a1c227e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:26,635 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:26,636 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=13, startTime=1732416446570; duration=0sec 2024-11-24T02:47:26,636 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:26,636 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:26,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742110_1286 (size=12561) 2024-11-24T02:47:26,644 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/6b3ac041a51b4a718c33c9feffae6241 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6b3ac041a51b4a718c33c9feffae6241 2024-11-24T02:47:26,649 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into 6b3ac041a51b4a718c33c9feffae6241(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:26,649 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:26,649 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=13, startTime=1732416446571; duration=0sec 2024-11-24T02:47:26,649 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:26,649 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:26,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T02:47:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:26,702 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:47:26,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/19e7fda9ab224ca58272bf144d35c5bb is 50, key is test_row_0/A:col10/1732416445696/Put/seqid=0 2024-11-24T02:47:26,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742111_1287 (size=12151) 2024-11-24T02:47:26,712 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/19e7fda9ab224ca58272bf144d35c5bb 2024-11-24T02:47:26,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/47910c94b7754be388ba42c115fc90b7 is 50, key is test_row_0/B:col10/1732416445696/Put/seqid=0 2024-11-24T02:47:26,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742112_1288 (size=12151) 2024-11-24T02:47:26,745 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/47910c94b7754be388ba42c115fc90b7 2024-11-24T02:47:26,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/c3517dbb2d9e4095aeee651389be3cf8 is 50, key is test_row_0/C:col10/1732416445696/Put/seqid=0 2024-11-24T02:47:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:47:26,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742113_1289 (size=12151) 2024-11-24T02:47:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:26,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:26,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416506870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416506872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416506872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416506873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416506882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416506986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416506986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416506987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416506987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,173 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/c3517dbb2d9e4095aeee651389be3cf8 2024-11-24T02:47:27,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/19e7fda9ab224ca58272bf144d35c5bb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/19e7fda9ab224ca58272bf144d35c5bb 2024-11-24T02:47:27,192 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/19e7fda9ab224ca58272bf144d35c5bb, entries=150, sequenceid=199, filesize=11.9 K 2024-11-24T02:47:27,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/47910c94b7754be388ba42c115fc90b7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/47910c94b7754be388ba42c115fc90b7 2024-11-24T02:47:27,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416507197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,199 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/47910c94b7754be388ba42c115fc90b7, entries=150, sequenceid=199, filesize=11.9 K 2024-11-24T02:47:27,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416507197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416507198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/c3517dbb2d9e4095aeee651389be3cf8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/c3517dbb2d9e4095aeee651389be3cf8 2024-11-24T02:47:27,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416507198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,206 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/c3517dbb2d9e4095aeee651389be3cf8, entries=150, sequenceid=199, filesize=11.9 K 2024-11-24T02:47:27,208 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9560919f24867d41b2f73bc8d1a4195 in 506ms, sequenceid=199, compaction requested=false 2024-11-24T02:47:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:27,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-24T02:47:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-24T02:47:27,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T02:47:27,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5880 sec 2024-11-24T02:47:27,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5940 sec 2024-11-24T02:47:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:27,508 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:27,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:47:27,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f320251ba634474b878501dfcbdc7d4b is 50, key is test_row_0/A:col10/1732416446871/Put/seqid=0 2024-11-24T02:47:27,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742114_1290 (size=12151) 2024-11-24T02:47:27,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416507539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416507548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416507548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416507553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416507657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416507657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416507657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416507666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T02:47:27,766 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:27,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-24T02:47:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:47:27,768 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:27,769 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:27,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:27,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416507866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416507866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416507867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:47:27,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416507876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416507887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:27,891 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:27,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:27,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:47:27,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:27,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:27,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:27,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:27,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:27,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f320251ba634474b878501dfcbdc7d4b 2024-11-24T02:47:27,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b889e0a2d2354f76a4330ba1f54948fc is 50, key is test_row_0/B:col10/1732416446871/Put/seqid=0 2024-11-24T02:47:27,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742115_1291 (size=12151) 2024-11-24T02:47:28,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:47:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:47:28,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416508177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416508177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416508177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416508187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:47:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b889e0a2d2354f76a4330ba1f54948fc 2024-11-24T02:47:28,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/66318575b15b46d7ad7c984b443c0f02 is 50, key is test_row_0/C:col10/1732416446871/Put/seqid=0 2024-11-24T02:47:28,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742116_1292 (size=12151) 2024-11-24T02:47:28,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/66318575b15b46d7ad7c984b443c0f02 2024-11-24T02:47:28,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f320251ba634474b878501dfcbdc7d4b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f320251ba634474b878501dfcbdc7d4b 2024-11-24T02:47:28,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:47:28,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:28,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f320251ba634474b878501dfcbdc7d4b, entries=150, sequenceid=212, filesize=11.9 K 2024-11-24T02:47:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b889e0a2d2354f76a4330ba1f54948fc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b889e0a2d2354f76a4330ba1f54948fc 2024-11-24T02:47:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:28,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b889e0a2d2354f76a4330ba1f54948fc, entries=150, sequenceid=212, filesize=11.9 K 2024-11-24T02:47:28,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/66318575b15b46d7ad7c984b443c0f02 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/66318575b15b46d7ad7c984b443c0f02 2024-11-24T02:47:28,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/66318575b15b46d7ad7c984b443c0f02, entries=150, sequenceid=212, filesize=11.9 K 2024-11-24T02:47:28,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for a9560919f24867d41b2f73bc8d1a4195 in 882ms, sequenceid=212, compaction requested=true 2024-11-24T02:47:28,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:28,391 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:28,391 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:28,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:28,398 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:47:28,398 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:28,398 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,399 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/18e8350a4c204bb3bbf6d8c9a1c227e0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/19e7fda9ab224ca58272bf144d35c5bb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f320251ba634474b878501dfcbdc7d4b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=36.0 K 2024-11-24T02:47:28,399 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:28,399 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:28,399 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,399 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/1933a1be6aa24d19bb6d59200152fa99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/47910c94b7754be388ba42c115fc90b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b889e0a2d2354f76a4330ba1f54948fc] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=36.0 K 2024-11-24T02:47:28,399 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18e8350a4c204bb3bbf6d8c9a1c227e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416445613 2024-11-24T02:47:28,399 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1933a1be6aa24d19bb6d59200152fa99, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416445613 2024-11-24T02:47:28,399 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19e7fda9ab224ca58272bf144d35c5bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732416445696 2024-11-24T02:47:28,400 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 47910c94b7754be388ba42c115fc90b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732416445696 2024-11-24T02:47:28,400 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f320251ba634474b878501dfcbdc7d4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732416446866 2024-11-24T02:47:28,400 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b889e0a2d2354f76a4330ba1f54948fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732416446866 2024-11-24T02:47:28,410 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#245 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:28,411 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:28,411 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/0231fa628cbc4303a9464472e0ef550a is 50, key is test_row_0/A:col10/1732416446871/Put/seqid=0 2024-11-24T02:47:28,411 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/72e1a8fa4fbf45ff9038b10445d4f1e7 is 50, key is test_row_0/B:col10/1732416446871/Put/seqid=0 2024-11-24T02:47:28,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742117_1293 (size=12663) 2024-11-24T02:47:28,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742118_1294 (size=12663) 2024-11-24T02:47:28,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T02:47:28,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,535 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:47:28,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/bba992cb5f154de69f017fc662b0975d is 50, key is test_row_0/A:col10/1732416447544/Put/seqid=0 2024-11-24T02:47:28,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742119_1295 (size=12151) 2024-11-24T02:47:28,546 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/bba992cb5f154de69f017fc662b0975d 2024-11-24T02:47:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/11e43d4be3ee4c69a0f2f473a2e4aeb5 is 50, key is test_row_0/B:col10/1732416447544/Put/seqid=0 2024-11-24T02:47:28,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742120_1296 (size=12151) 2024-11-24T02:47:28,560 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/11e43d4be3ee4c69a0f2f473a2e4aeb5 2024-11-24T02:47:28,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/7a1c7d262e644165827c85c249096b72 is 50, key is test_row_0/C:col10/1732416447544/Put/seqid=0 2024-11-24T02:47:28,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742121_1297 (size=12151) 2024-11-24T02:47:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:28,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:28,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416508697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416508700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416508701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416508702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416508807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416508807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416508816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416508816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:28,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:28,822 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/72e1a8fa4fbf45ff9038b10445d4f1e7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/72e1a8fa4fbf45ff9038b10445d4f1e7 2024-11-24T02:47:28,822 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/0231fa628cbc4303a9464472e0ef550a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/0231fa628cbc4303a9464472e0ef550a 2024-11-24T02:47:28,833 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into 72e1a8fa4fbf45ff9038b10445d4f1e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:28,833 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:28,833 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=13, startTime=1732416448391; duration=0sec 2024-11-24T02:47:28,833 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:28,834 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:28,834 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:28,834 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into 0231fa628cbc4303a9464472e0ef550a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:28,834 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:28,834 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=13, startTime=1732416448390; duration=0sec 2024-11-24T02:47:28,834 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:28,834 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:28,835 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:28,835 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:28,835 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,835 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6b3ac041a51b4a718c33c9feffae6241, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/c3517dbb2d9e4095aeee651389be3cf8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/66318575b15b46d7ad7c984b443c0f02] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=36.0 K 2024-11-24T02:47:28,835 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b3ac041a51b4a718c33c9feffae6241, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732416445613 2024-11-24T02:47:28,836 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c3517dbb2d9e4095aeee651389be3cf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732416445696 2024-11-24T02:47:28,836 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 66318575b15b46d7ad7c984b443c0f02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732416446866 2024-11-24T02:47:28,846 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:28,846 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/fea34f3ec1854fc28522e4833e19fa82 is 50, key is test_row_0/C:col10/1732416446871/Put/seqid=0 2024-11-24T02:47:28,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742122_1298 (size=12663) 2024-11-24T02:47:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:47:28,972 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/7a1c7d262e644165827c85c249096b72 2024-11-24T02:47:28,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/bba992cb5f154de69f017fc662b0975d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/bba992cb5f154de69f017fc662b0975d 2024-11-24T02:47:28,979 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/bba992cb5f154de69f017fc662b0975d, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T02:47:28,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/11e43d4be3ee4c69a0f2f473a2e4aeb5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/11e43d4be3ee4c69a0f2f473a2e4aeb5 2024-11-24T02:47:28,985 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/11e43d4be3ee4c69a0f2f473a2e4aeb5, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T02:47:28,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/7a1c7d262e644165827c85c249096b72 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/7a1c7d262e644165827c85c249096b72 2024-11-24T02:47:28,996 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/7a1c7d262e644165827c85c249096b72, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T02:47:28,997 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9560919f24867d41b2f73bc8d1a4195 in 461ms, sequenceid=237, compaction requested=false 2024-11-24T02:47:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-24T02:47:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-24T02:47:29,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T02:47:29,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2290 sec 2024-11-24T02:47:29,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.2350 sec 2024-11-24T02:47:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:29,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:29,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:47:29,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f0bdd55c5b4f49b5aeffbb9fda9605e9 is 50, key is test_row_0/A:col10/1732416448700/Put/seqid=0 2024-11-24T02:47:29,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742123_1299 (size=12147) 2024-11-24T02:47:29,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f0bdd55c5b4f49b5aeffbb9fda9605e9 2024-11-24T02:47:29,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/67f1405817fd45a3a93eae2b55cc139b is 50, key is test_row_0/B:col10/1732416448700/Put/seqid=0 2024-11-24T02:47:29,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742124_1300 (size=9757) 2024-11-24T02:47:29,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/67f1405817fd45a3a93eae2b55cc139b 2024-11-24T02:47:29,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416509053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416509054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416509054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416509055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/3f36a2603fe64f56975f05660815177b is 50, key is test_row_0/C:col10/1732416448700/Put/seqid=0 2024-11-24T02:47:29,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742125_1301 (size=9757) 2024-11-24T02:47:29,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416509166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416509167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416509167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416509167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,266 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/fea34f3ec1854fc28522e4833e19fa82 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/fea34f3ec1854fc28522e4833e19fa82 2024-11-24T02:47:29,275 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into fea34f3ec1854fc28522e4833e19fa82(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:29,275 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:29,275 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=13, startTime=1732416448391; duration=0sec 2024-11-24T02:47:29,275 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:29,275 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:29,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416509377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416509376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416509377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416509377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/3f36a2603fe64f56975f05660815177b 2024-11-24T02:47:29,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f0bdd55c5b4f49b5aeffbb9fda9605e9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f0bdd55c5b4f49b5aeffbb9fda9605e9 2024-11-24T02:47:29,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f0bdd55c5b4f49b5aeffbb9fda9605e9, entries=150, sequenceid=252, filesize=11.9 K 2024-11-24T02:47:29,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/67f1405817fd45a3a93eae2b55cc139b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/67f1405817fd45a3a93eae2b55cc139b 2024-11-24T02:47:29,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/67f1405817fd45a3a93eae2b55cc139b, entries=100, sequenceid=252, filesize=9.5 K 2024-11-24T02:47:29,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/3f36a2603fe64f56975f05660815177b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3f36a2603fe64f56975f05660815177b 2024-11-24T02:47:29,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3f36a2603fe64f56975f05660815177b, entries=100, sequenceid=252, filesize=9.5 K 2024-11-24T02:47:29,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a9560919f24867d41b2f73bc8d1a4195 in 479ms, sequenceid=252, compaction requested=true 2024-11-24T02:47:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:29,498 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:29,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:47:29,498 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:29,499 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36961 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:29,499 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:29,499 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:29,499 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:29,499 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:29,499 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:29,499 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/72e1a8fa4fbf45ff9038b10445d4f1e7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/11e43d4be3ee4c69a0f2f473a2e4aeb5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/67f1405817fd45a3a93eae2b55cc139b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=33.8 K 2024-11-24T02:47:29,499 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/0231fa628cbc4303a9464472e0ef550a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/bba992cb5f154de69f017fc662b0975d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f0bdd55c5b4f49b5aeffbb9fda9605e9] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=36.1 K 2024-11-24T02:47:29,500 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0231fa628cbc4303a9464472e0ef550a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732416446866 2024-11-24T02:47:29,500 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72e1a8fa4fbf45ff9038b10445d4f1e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732416446866 2024-11-24T02:47:29,500 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11e43d4be3ee4c69a0f2f473a2e4aeb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732416447544 2024-11-24T02:47:29,500 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting bba992cb5f154de69f017fc662b0975d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732416447544 2024-11-24T02:47:29,501 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f1405817fd45a3a93eae2b55cc139b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416448700 2024-11-24T02:47:29,501 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f0bdd55c5b4f49b5aeffbb9fda9605e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416448700 2024-11-24T02:47:29,509 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:29,510 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/3f797d7498c54ce3889ff84ba3a494dc is 50, key is test_row_0/B:col10/1732416448700/Put/seqid=0 2024-11-24T02:47:29,510 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:29,510 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/a1e96e0e4e854f14b637c4ea4961a00c is 50, key is test_row_0/A:col10/1732416448700/Put/seqid=0 2024-11-24T02:47:29,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742127_1303 (size=12765) 2024-11-24T02:47:29,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742126_1302 (size=12765) 2024-11-24T02:47:29,543 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/3f797d7498c54ce3889ff84ba3a494dc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/3f797d7498c54ce3889ff84ba3a494dc 2024-11-24T02:47:29,548 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into 3f797d7498c54ce3889ff84ba3a494dc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:29,548 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:29,548 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=13, startTime=1732416449498; duration=0sec 2024-11-24T02:47:29,549 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:29,549 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:29,549 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:29,550 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34571 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:29,550 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:29,550 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:29,550 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/fea34f3ec1854fc28522e4833e19fa82, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/7a1c7d262e644165827c85c249096b72, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3f36a2603fe64f56975f05660815177b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=33.8 K 2024-11-24T02:47:29,551 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting fea34f3ec1854fc28522e4833e19fa82, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732416446866 2024-11-24T02:47:29,551 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a1c7d262e644165827c85c249096b72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732416447544 2024-11-24T02:47:29,551 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f36a2603fe64f56975f05660815177b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416448700 2024-11-24T02:47:29,563 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:29,564 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/4397ca8909c8496d991159a5e8f7648a is 50, key is test_row_0/C:col10/1732416448700/Put/seqid=0 2024-11-24T02:47:29,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742128_1304 (size=12765) 2024-11-24T02:47:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:29,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:29,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:47:29,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/c5bb8445d1374042b7cda8ba7ca1f16d is 50, key is test_row_0/A:col10/1732416449689/Put/seqid=0 2024-11-24T02:47:29,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416509699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742129_1305 (size=14741) 2024-11-24T02:47:29,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416509701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416509702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416509703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416509806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416509817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416509817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416509817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416509907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:29,908 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T02:47:29,917 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:29,918 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-24T02:47:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:47:29,919 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:29,920 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:29,920 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:29,934 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/a1e96e0e4e854f14b637c4ea4961a00c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/a1e96e0e4e854f14b637c4ea4961a00c 2024-11-24T02:47:29,938 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into a1e96e0e4e854f14b637c4ea4961a00c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:29,938 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:29,938 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=13, startTime=1732416449498; duration=0sec 2024-11-24T02:47:29,938 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:29,938 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:29,994 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/4397ca8909c8496d991159a5e8f7648a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4397ca8909c8496d991159a5e8f7648a 2024-11-24T02:47:29,999 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into 4397ca8909c8496d991159a5e8f7648a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:29,999 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:29,999 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=13, startTime=1732416449498; duration=0sec 2024-11-24T02:47:29,999 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:29,999 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416510017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:47:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416510026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416510026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416510026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:47:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:30,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/c5bb8445d1374042b7cda8ba7ca1f16d 2024-11-24T02:47:30,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/4f9d489530584b09b2b36f4d51cfcec4 is 50, key is test_row_0/B:col10/1732416449689/Put/seqid=0 2024-11-24T02:47:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742130_1306 (size=12301) 2024-11-24T02:47:30,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:47:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:47:30,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416510326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416510337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416510337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416510337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:47:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/4f9d489530584b09b2b36f4d51cfcec4 2024-11-24T02:47:30,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/a204c01a6c6c4132826aa4df6573b2a1 is 50, key is test_row_0/C:col10/1732416449689/Put/seqid=0 2024-11-24T02:47:30,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:47:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:30,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742131_1307 (size=12301) 2024-11-24T02:47:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:47:30,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/a204c01a6c6c4132826aa4df6573b2a1 2024-11-24T02:47:30,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/c5bb8445d1374042b7cda8ba7ca1f16d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/c5bb8445d1374042b7cda8ba7ca1f16d 2024-11-24T02:47:30,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/c5bb8445d1374042b7cda8ba7ca1f16d, entries=200, sequenceid=280, filesize=14.4 K 2024-11-24T02:47:30,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/4f9d489530584b09b2b36f4d51cfcec4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/4f9d489530584b09b2b36f4d51cfcec4 2024-11-24T02:47:30,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/4f9d489530584b09b2b36f4d51cfcec4, entries=150, sequenceid=280, filesize=12.0 K 2024-11-24T02:47:30,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/a204c01a6c6c4132826aa4df6573b2a1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a204c01a6c6c4132826aa4df6573b2a1 2024-11-24T02:47:30,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a204c01a6c6c4132826aa4df6573b2a1, entries=150, sequenceid=280, filesize=12.0 K 2024-11-24T02:47:30,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for a9560919f24867d41b2f73bc8d1a4195 in 881ms, sequenceid=280, compaction requested=false 2024-11-24T02:47:30,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:30,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T02:47:30,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:30,687 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:30,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f59b51023f304e98a0d6d295a2507b4d is 50, key is test_row_0/A:col10/1732416449701/Put/seqid=0 2024-11-24T02:47:30,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742132_1308 (size=12301) 2024-11-24T02:47:30,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f59b51023f304e98a0d6d295a2507b4d 2024-11-24T02:47:30,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/0fd2de142f694b9a9778d4c8273d9d14 is 50, key is test_row_0/B:col10/1732416449701/Put/seqid=0 2024-11-24T02:47:30,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742133_1309 (size=12301) 2024-11-24T02:47:30,780 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/0fd2de142f694b9a9778d4c8273d9d14 2024-11-24T02:47:30,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0f965c75a1884187af3f7a4912b1a6d9 is 50, key is test_row_0/C:col10/1732416449701/Put/seqid=0 2024-11-24T02:47:30,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:30,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742134_1310 (size=12301) 2024-11-24T02:47:30,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416510893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416510894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416510894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:30,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416510900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416511006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416511006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416511006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416511016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:47:31,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416511216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416511216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416511216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416511226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,251 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0f965c75a1884187af3f7a4912b1a6d9 2024-11-24T02:47:31,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/f59b51023f304e98a0d6d295a2507b4d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f59b51023f304e98a0d6d295a2507b4d 2024-11-24T02:47:31,258 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f59b51023f304e98a0d6d295a2507b4d, entries=150, sequenceid=293, filesize=12.0 K 2024-11-24T02:47:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/0fd2de142f694b9a9778d4c8273d9d14 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/0fd2de142f694b9a9778d4c8273d9d14 2024-11-24T02:47:31,262 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/0fd2de142f694b9a9778d4c8273d9d14, entries=150, sequenceid=293, filesize=12.0 K 2024-11-24T02:47:31,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0f965c75a1884187af3f7a4912b1a6d9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0f965c75a1884187af3f7a4912b1a6d9 2024-11-24T02:47:31,266 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0f965c75a1884187af3f7a4912b1a6d9, entries=150, sequenceid=293, filesize=12.0 K 2024-11-24T02:47:31,267 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9560919f24867d41b2f73bc8d1a4195 in 581ms, sequenceid=293, compaction requested=true 2024-11-24T02:47:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:31,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-24T02:47:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-24T02:47:31,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-24T02:47:31,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3480 sec 2024-11-24T02:47:31,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3520 sec 2024-11-24T02:47:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:31,529 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:31,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T02:47:31,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/7f8e223aaebb4322892b7489fa4fffaf is 50, key is test_row_0/A:col10/1732416451528/Put/seqid=0 2024-11-24T02:47:31,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742135_1311 (size=14741) 2024-11-24T02:47:31,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416511536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416511541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416511542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416511542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416511646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416511646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416511657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416511657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416511857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416511857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416511867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416511867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:31,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:31,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/7f8e223aaebb4322892b7489fa4fffaf 2024-11-24T02:47:31,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/ab6f33d8b7f24d878b5e5270820d784d is 50, key is test_row_0/B:col10/1732416451528/Put/seqid=0 2024-11-24T02:47:31,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742136_1312 (size=12301) 2024-11-24T02:47:31,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/ab6f33d8b7f24d878b5e5270820d784d 2024-11-24T02:47:31,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/bc891a6034f9438c88a711cc98ff8ed1 is 50, key is test_row_0/C:col10/1732416451528/Put/seqid=0 2024-11-24T02:47:31,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742137_1313 (size=12301) 2024-11-24T02:47:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T02:47:32,067 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:32,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-24T02:47:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:47:32,072 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:32,073 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:32,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416512166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416512166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:47:32,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416512177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416512178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:47:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:32,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:47:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:32,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:47:32,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/bc891a6034f9438c88a711cc98ff8ed1 2024-11-24T02:47:32,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/7f8e223aaebb4322892b7489fa4fffaf as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f8e223aaebb4322892b7489fa4fffaf 2024-11-24T02:47:32,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f8e223aaebb4322892b7489fa4fffaf, entries=200, sequenceid=319, filesize=14.4 K 2024-11-24T02:47:32,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/ab6f33d8b7f24d878b5e5270820d784d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/ab6f33d8b7f24d878b5e5270820d784d 2024-11-24T02:47:32,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/ab6f33d8b7f24d878b5e5270820d784d, entries=150, sequenceid=319, filesize=12.0 K 2024-11-24T02:47:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/bc891a6034f9438c88a711cc98ff8ed1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/bc891a6034f9438c88a711cc98ff8ed1 2024-11-24T02:47:32,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/bc891a6034f9438c88a711cc98ff8ed1, entries=150, sequenceid=319, filesize=12.0 K 2024-11-24T02:47:32,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for a9560919f24867d41b2f73bc8d1a4195 in 900ms, sequenceid=319, compaction requested=true 2024-11-24T02:47:32,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:32,429 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:32,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:32,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:32,430 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:32,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:32,430 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:32,430 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:32,430 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:32,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:32,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:32,431 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/a1e96e0e4e854f14b637c4ea4961a00c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/c5bb8445d1374042b7cda8ba7ca1f16d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f59b51023f304e98a0d6d295a2507b4d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f8e223aaebb4322892b7489fa4fffaf] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=53.3 K 2024-11-24T02:47:32,431 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1e96e0e4e854f14b637c4ea4961a00c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416447545 2024-11-24T02:47:32,431 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:32,431 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:32,431 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,431 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/3f797d7498c54ce3889ff84ba3a494dc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/4f9d489530584b09b2b36f4d51cfcec4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/0fd2de142f694b9a9778d4c8273d9d14, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/ab6f33d8b7f24d878b5e5270820d784d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=48.5 K 2024-11-24T02:47:32,432 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5bb8445d1374042b7cda8ba7ca1f16d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732416449037 2024-11-24T02:47:32,432 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f797d7498c54ce3889ff84ba3a494dc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416447545 2024-11-24T02:47:32,432 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f59b51023f304e98a0d6d295a2507b4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732416449700 2024-11-24T02:47:32,432 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f8e223aaebb4322892b7489fa4fffaf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732416450870 2024-11-24T02:47:32,433 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f9d489530584b09b2b36f4d51cfcec4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732416449689 2024-11-24T02:47:32,433 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fd2de142f694b9a9778d4c8273d9d14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732416449700 2024-11-24T02:47:32,434 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6f33d8b7f24d878b5e5270820d784d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732416450892 2024-11-24T02:47:32,459 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:32,460 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/1e2b553bf710473cb3ca519050117c03 is 50, key is test_row_0/A:col10/1732416451528/Put/seqid=0 2024-11-24T02:47:32,469 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:32,469 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/d1920b664cce44af9ccc9778960804d9 is 50, key is test_row_0/B:col10/1732416451528/Put/seqid=0 2024-11-24T02:47:32,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742138_1314 (size=13051) 2024-11-24T02:47:32,496 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/1e2b553bf710473cb3ca519050117c03 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/1e2b553bf710473cb3ca519050117c03 2024-11-24T02:47:32,503 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into 1e2b553bf710473cb3ca519050117c03(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:32,503 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:32,504 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=12, startTime=1732416452429; duration=0sec 2024-11-24T02:47:32,504 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:32,504 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:32,504 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:32,507 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:32,507 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:32,508 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,508 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4397ca8909c8496d991159a5e8f7648a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a204c01a6c6c4132826aa4df6573b2a1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0f965c75a1884187af3f7a4912b1a6d9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/bc891a6034f9438c88a711cc98ff8ed1] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=48.5 K 2024-11-24T02:47:32,508 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4397ca8909c8496d991159a5e8f7648a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732416447545 2024-11-24T02:47:32,508 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a204c01a6c6c4132826aa4df6573b2a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1732416449689 2024-11-24T02:47:32,509 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f965c75a1884187af3f7a4912b1a6d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732416449700 2024-11-24T02:47:32,509 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc891a6034f9438c88a711cc98ff8ed1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732416450892 2024-11-24T02:47:32,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742139_1315 (size=13051) 2024-11-24T02:47:32,525 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:32,526 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/ce763f14700645c0a0210eb2f79be2e0 is 50, key is test_row_0/C:col10/1732416451528/Put/seqid=0 2024-11-24T02:47:32,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T02:47:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:32,532 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:32,537 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/d1920b664cce44af9ccc9778960804d9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/d1920b664cce44af9ccc9778960804d9 2024-11-24T02:47:32,544 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into d1920b664cce44af9ccc9778960804d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:32,544 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:32,544 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=12, startTime=1732416452430; duration=0sec 2024-11-24T02:47:32,544 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:32,544 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:32,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ae02a541fd194c759f84f01749c776e7 is 50, key is test_row_0/A:col10/1732416451541/Put/seqid=0 2024-11-24T02:47:32,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742140_1316 (size=13051) 2024-11-24T02:47:32,578 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/ce763f14700645c0a0210eb2f79be2e0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ce763f14700645c0a0210eb2f79be2e0 2024-11-24T02:47:32,589 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into ce763f14700645c0a0210eb2f79be2e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:32,589 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:32,589 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=12, startTime=1732416452431; duration=0sec 2024-11-24T02:47:32,589 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:32,589 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:32,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742141_1317 (size=12301) 2024-11-24T02:47:32,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:32,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:47:32,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416512705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416512705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416512707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416512708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416512817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416512818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416512818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416512818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:32,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,005 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ae02a541fd194c759f84f01749c776e7 2024-11-24T02:47:33,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/132615f29d3140bdb8585720d74086f9 is 50, key is test_row_0/B:col10/1732416451541/Put/seqid=0 2024-11-24T02:47:33,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416513027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416513027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416513027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416513027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742142_1318 (size=12301) 2024-11-24T02:47:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:47:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416513336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416513336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416513337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416513337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,441 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/132615f29d3140bdb8585720d74086f9 2024-11-24T02:47:33,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7 is 50, key is test_row_0/C:col10/1732416451541/Put/seqid=0 2024-11-24T02:47:33,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742143_1319 (size=12301) 2024-11-24T02:47:33,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416513846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416513846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416513847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416513847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:33,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:33,852 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7 2024-11-24T02:47:33,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ae02a541fd194c759f84f01749c776e7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ae02a541fd194c759f84f01749c776e7 2024-11-24T02:47:33,861 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ae02a541fd194c759f84f01749c776e7, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:47:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/132615f29d3140bdb8585720d74086f9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/132615f29d3140bdb8585720d74086f9 2024-11-24T02:47:33,865 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/132615f29d3140bdb8585720d74086f9, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:47:33,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7 2024-11-24T02:47:33,874 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:47:33,876 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9560919f24867d41b2f73bc8d1a4195 in 1344ms, sequenceid=331, compaction requested=false 2024-11-24T02:47:33,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:33,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:33,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-24T02:47:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-24T02:47:33,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T02:47:33,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8050 sec 2024-11-24T02:47:33,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8140 sec 2024-11-24T02:47:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:33,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:33,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T02:47:33,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/b06e079c01614d50b0385a0066de833c is 50, key is test_row_0/A:col10/1732416452703/Put/seqid=0 2024-11-24T02:47:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742144_1320 (size=14741) 2024-11-24T02:47:33,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/b06e079c01614d50b0385a0066de833c 2024-11-24T02:47:33,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/dec27c94b00a44f28a8263cc713c5e03 is 50, key is test_row_0/B:col10/1732416452703/Put/seqid=0 2024-11-24T02:47:34,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416513998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742145_1321 (size=12301) 2024-11-24T02:47:34,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/dec27c94b00a44f28a8263cc713c5e03 2024-11-24T02:47:34,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/39b8f096feb243aea34438d89ff6ed5b is 50, key is test_row_0/C:col10/1732416452703/Put/seqid=0 2024-11-24T02:47:34,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742146_1322 (size=12301) 2024-11-24T02:47:34,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/39b8f096feb243aea34438d89ff6ed5b 2024-11-24T02:47:34,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/b06e079c01614d50b0385a0066de833c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b06e079c01614d50b0385a0066de833c 2024-11-24T02:47:34,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b06e079c01614d50b0385a0066de833c, entries=200, sequenceid=359, filesize=14.4 K 2024-11-24T02:47:34,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/dec27c94b00a44f28a8263cc713c5e03 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/dec27c94b00a44f28a8263cc713c5e03 2024-11-24T02:47:34,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/dec27c94b00a44f28a8263cc713c5e03, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:47:34,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/39b8f096feb243aea34438d89ff6ed5b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/39b8f096feb243aea34438d89ff6ed5b 2024-11-24T02:47:34,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/39b8f096feb243aea34438d89ff6ed5b, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:47:34,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a9560919f24867d41b2f73bc8d1a4195 in 155ms, sequenceid=359, compaction requested=true 2024-11-24T02:47:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:34,105 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:34,105 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:34,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:34,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:34,107 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:34,107 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:34,107 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:34,107 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/d1920b664cce44af9ccc9778960804d9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/132615f29d3140bdb8585720d74086f9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/dec27c94b00a44f28a8263cc713c5e03] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=36.8 K 2024-11-24T02:47:34,107 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:34,107 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:34,107 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:34,107 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/1e2b553bf710473cb3ca519050117c03, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ae02a541fd194c759f84f01749c776e7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b06e079c01614d50b0385a0066de833c] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=39.2 K 2024-11-24T02:47:34,109 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e2b553bf710473cb3ca519050117c03, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732416450892 2024-11-24T02:47:34,110 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d1920b664cce44af9ccc9778960804d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732416450892 2024-11-24T02:47:34,110 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 132615f29d3140bdb8585720d74086f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416451535 2024-11-24T02:47:34,110 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae02a541fd194c759f84f01749c776e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416451535 2024-11-24T02:47:34,111 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting dec27c94b00a44f28a8263cc713c5e03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416452703 2024-11-24T02:47:34,111 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06e079c01614d50b0385a0066de833c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416452703 2024-11-24T02:47:34,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:34,120 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:34,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:34,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/447e686fc070406ca0bb5b40622de7e2 is 50, key is test_row_0/A:col10/1732416453958/Put/seqid=0 2024-11-24T02:47:34,142 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:34,142 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b763d43461a645838ebb6151442736df is 50, key is test_row_0/B:col10/1732416452703/Put/seqid=0 2024-11-24T02:47:34,157 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:34,158 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ec4ae13e917144479517ef173e1b310b is 50, key is test_row_0/A:col10/1732416452703/Put/seqid=0 2024-11-24T02:47:34,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742147_1323 (size=14741) 2024-11-24T02:47:34,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/447e686fc070406ca0bb5b40622de7e2 2024-11-24T02:47:34,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742149_1325 (size=13153) 2024-11-24T02:47:34,201 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/ec4ae13e917144479517ef173e1b310b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ec4ae13e917144479517ef173e1b310b 2024-11-24T02:47:34,207 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into ec4ae13e917144479517ef173e1b310b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:34,207 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:34,207 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=13, startTime=1732416454105; duration=0sec 2024-11-24T02:47:34,207 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:34,207 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:34,208 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:34,209 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:34,209 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:34,209 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:34,209 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ce763f14700645c0a0210eb2f79be2e0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/39b8f096feb243aea34438d89ff6ed5b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=36.8 K 2024-11-24T02:47:34,209 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce763f14700645c0a0210eb2f79be2e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732416450892 2024-11-24T02:47:34,210 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d7f0bfd8704b8fa22ef0e0bd2d53b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416451535 2024-11-24T02:47:34,210 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b8f096feb243aea34438d89ff6ed5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416452703 2024-11-24T02:47:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T02:47:34,217 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:34,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742148_1324 (size=13153) 2024-11-24T02:47:34,219 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:34,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-24T02:47:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:47:34,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:34,222 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:34,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:34,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2b25f8f94dd04e118b58e5703b47994a is 50, key is test_row_0/B:col10/1732416453958/Put/seqid=0 2024-11-24T02:47:34,230 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:34,230 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/643d9458a232430c918f69b0897e767f is 50, key is test_row_0/C:col10/1732416452703/Put/seqid=0 2024-11-24T02:47:34,233 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/b763d43461a645838ebb6151442736df as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b763d43461a645838ebb6151442736df 2024-11-24T02:47:34,242 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into b763d43461a645838ebb6151442736df(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:34,242 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:34,242 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=13, startTime=1732416454105; duration=0sec 2024-11-24T02:47:34,242 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:34,242 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:34,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742150_1326 (size=12301) 2024-11-24T02:47:34,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2b25f8f94dd04e118b58e5703b47994a 2024-11-24T02:47:34,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742151_1327 (size=13153) 2024-11-24T02:47:34,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/e9a3b4657a8e4b44b93bd027ff1a05b7 is 50, key is test_row_0/C:col10/1732416453958/Put/seqid=0 2024-11-24T02:47:34,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742152_1328 (size=12301) 2024-11-24T02:47:34,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/e9a3b4657a8e4b44b93bd027ff1a05b7 2024-11-24T02:47:34,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416514289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/447e686fc070406ca0bb5b40622de7e2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/447e686fc070406ca0bb5b40622de7e2 2024-11-24T02:47:34,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/447e686fc070406ca0bb5b40622de7e2, entries=200, sequenceid=370, filesize=14.4 K 2024-11-24T02:47:34,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2b25f8f94dd04e118b58e5703b47994a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2b25f8f94dd04e118b58e5703b47994a 2024-11-24T02:47:34,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2b25f8f94dd04e118b58e5703b47994a, entries=150, sequenceid=370, filesize=12.0 K 2024-11-24T02:47:34,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/e9a3b4657a8e4b44b93bd027ff1a05b7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/e9a3b4657a8e4b44b93bd027ff1a05b7 2024-11-24T02:47:34,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/e9a3b4657a8e4b44b93bd027ff1a05b7, entries=150, sequenceid=370, filesize=12.0 K 2024-11-24T02:47:34,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9560919f24867d41b2f73bc8d1a4195 in 191ms, sequenceid=370, compaction requested=false 2024-11-24T02:47:34,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:47:34,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T02:47:34,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:34,375 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:47:34,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/7f3cb58712bb494d848b00de5964533c is 50, key is test_row_0/A:col10/1732416454284/Put/seqid=0 2024-11-24T02:47:34,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742153_1329 (size=12301) 2024-11-24T02:47:34,383 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/7f3cb58712bb494d848b00de5964533c 2024-11-24T02:47:34,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2016b7b8f9b84fbca8ec1270d304514c is 50, key is test_row_0/B:col10/1732416454284/Put/seqid=0 2024-11-24T02:47:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742154_1330 (size=12301) 2024-11-24T02:47:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:34,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:34,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416514440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:47:34,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416514547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,652 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/643d9458a232430c918f69b0897e767f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/643d9458a232430c918f69b0897e767f 2024-11-24T02:47:34,656 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into 643d9458a232430c918f69b0897e767f(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:34,656 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:34,657 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=13, startTime=1732416454105; duration=0sec 2024-11-24T02:47:34,657 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:34,657 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:34,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416514756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,795 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2016b7b8f9b84fbca8ec1270d304514c 2024-11-24T02:47:34,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0fc6fbd426f149d187fcf03a60bef21c is 50, key is test_row_0/C:col10/1732416454284/Put/seqid=0 2024-11-24T02:47:34,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742155_1331 (size=12301) 2024-11-24T02:47:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:47:34,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416514856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416514856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,859 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416514856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,859 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,859 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:34,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416514867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:34,869 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:34,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:35,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416515066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:35,205 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0fc6fbd426f149d187fcf03a60bef21c 2024-11-24T02:47:35,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/7f3cb58712bb494d848b00de5964533c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f3cb58712bb494d848b00de5964533c 2024-11-24T02:47:35,212 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f3cb58712bb494d848b00de5964533c, entries=150, sequenceid=397, filesize=12.0 K 2024-11-24T02:47:35,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/2016b7b8f9b84fbca8ec1270d304514c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2016b7b8f9b84fbca8ec1270d304514c 2024-11-24T02:47:35,215 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2016b7b8f9b84fbca8ec1270d304514c, entries=150, sequenceid=397, filesize=12.0 K 2024-11-24T02:47:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/0fc6fbd426f149d187fcf03a60bef21c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0fc6fbd426f149d187fcf03a60bef21c 2024-11-24T02:47:35,219 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0fc6fbd426f149d187fcf03a60bef21c, entries=150, sequenceid=397, filesize=12.0 K 2024-11-24T02:47:35,220 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9560919f24867d41b2f73bc8d1a4195 in 845ms, sequenceid=397, compaction requested=true 2024-11-24T02:47:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-24T02:47:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-24T02:47:35,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-24T02:47:35,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0000 sec 2024-11-24T02:47:35,225 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0050 sec 2024-11-24T02:47:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T02:47:35,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:35,357 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-24T02:47:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:47:35,359 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:35,359 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:35,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:47:35,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T02:47:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:35,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:35,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/b0a6625297a341f19f72c0a57e1ca6a7 is 50, key is test_row_0/A:col10/1732416454439/Put/seqid=0 2024-11-24T02:47:35,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742156_1332 (size=12301) 2024-11-24T02:47:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:35,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:35,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416515671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:35,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:47:35,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416515787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:35,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:35,922 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/b0a6625297a341f19f72c0a57e1ca6a7 2024-11-24T02:47:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/46bdc9c8c4b5449d8dffd980a770eece is 50, key is test_row_0/B:col10/1732416454439/Put/seqid=0 2024-11-24T02:47:35,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742157_1333 (size=12301) 2024-11-24T02:47:35,933 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/46bdc9c8c4b5449d8dffd980a770eece 2024-11-24T02:47:35,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/172877a0e8fe402b9b98acb285b3893a is 50, key is test_row_0/C:col10/1732416454439/Put/seqid=0 2024-11-24T02:47:35,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742158_1334 (size=12301) 2024-11-24T02:47:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:47:36,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416515997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:36,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416516306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:36,345 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/172877a0e8fe402b9b98acb285b3893a 2024-11-24T02:47:36,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/b0a6625297a341f19f72c0a57e1ca6a7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b0a6625297a341f19f72c0a57e1ca6a7 2024-11-24T02:47:36,352 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b0a6625297a341f19f72c0a57e1ca6a7, entries=150, sequenceid=409, filesize=12.0 K 2024-11-24T02:47:36,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/46bdc9c8c4b5449d8dffd980a770eece as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/46bdc9c8c4b5449d8dffd980a770eece 2024-11-24T02:47:36,357 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/46bdc9c8c4b5449d8dffd980a770eece, entries=150, sequenceid=409, filesize=12.0 K 2024-11-24T02:47:36,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/172877a0e8fe402b9b98acb285b3893a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/172877a0e8fe402b9b98acb285b3893a 2024-11-24T02:47:36,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/172877a0e8fe402b9b98acb285b3893a, entries=150, sequenceid=409, filesize=12.0 K 2024-11-24T02:47:36,361 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9560919f24867d41b2f73bc8d1a4195 in 849ms, sequenceid=409, compaction requested=true 2024-11-24T02:47:36,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:36,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-24T02:47:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-24T02:47:36,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T02:47:36,364 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0040 sec 2024-11-24T02:47:36,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0080 sec 2024-11-24T02:47:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T02:47:36,497 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:36,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-24T02:47:36,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T02:47:36,499 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:36,500 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:36,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T02:47:36,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T02:47:36,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:36,652 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:47:36,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/457a3205f96c464ea3ff65bcdc3077eb is 50, key is test_row_0/A:col10/1732416455665/Put/seqid=0 2024-11-24T02:47:36,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742159_1335 (size=12301) 2024-11-24T02:47:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T02:47:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:36,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:36,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416516852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47876 deadline: 1732416516867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47964 deadline: 1732416516867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,870 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:36,870 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:36,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47938 deadline: 1732416516877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,881 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:36,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1732416516886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,891 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in a9560919f24867d41b2f73bc8d1a4195 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:36,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416516967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:37,060 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/457a3205f96c464ea3ff65bcdc3077eb 2024-11-24T02:47:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/f685adaea6854ae5a8f6ba6f543644cd is 50, key is test_row_0/B:col10/1732416455665/Put/seqid=0 2024-11-24T02:47:37,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742160_1336 (size=12301) 2024-11-24T02:47:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T02:47:37,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416517177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:37,469 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/f685adaea6854ae5a8f6ba6f543644cd 2024-11-24T02:47:37,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/a84ae41af7bd4385bf19cc47c8f8b099 is 50, key is test_row_0/C:col10/1732416455665/Put/seqid=0 2024-11-24T02:47:37,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742161_1337 (size=12301) 2024-11-24T02:47:37,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416517486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:37,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T02:47:37,881 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/a84ae41af7bd4385bf19cc47c8f8b099 2024-11-24T02:47:37,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/457a3205f96c464ea3ff65bcdc3077eb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/457a3205f96c464ea3ff65bcdc3077eb 2024-11-24T02:47:37,889 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/457a3205f96c464ea3ff65bcdc3077eb, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T02:47:37,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/f685adaea6854ae5a8f6ba6f543644cd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/f685adaea6854ae5a8f6ba6f543644cd 2024-11-24T02:47:37,892 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/f685adaea6854ae5a8f6ba6f543644cd, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T02:47:37,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/a84ae41af7bd4385bf19cc47c8f8b099 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a84ae41af7bd4385bf19cc47c8f8b099 2024-11-24T02:47:37,896 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a84ae41af7bd4385bf19cc47c8f8b099, entries=150, sequenceid=434, filesize=12.0 K 2024-11-24T02:47:37,897 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9560919f24867d41b2f73bc8d1a4195 in 1245ms, sequenceid=434, compaction requested=true 2024-11-24T02:47:37,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:37,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:37,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-24T02:47:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-24T02:47:37,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T02:47:37,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-24T02:47:37,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.4030 sec 2024-11-24T02:47:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:38,000 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:38,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:47:38,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/caa4ec3fc2e94eec998425aa4d508d79 is 50, key is test_row_0/A:col10/1732416456846/Put/seqid=0 2024-11-24T02:47:38,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742162_1338 (size=14741) 2024-11-24T02:47:38,105 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:47:38,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416518107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:38,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416518217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:38,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/caa4ec3fc2e94eec998425aa4d508d79 2024-11-24T02:47:38,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/e6ffe64ba8c340bf804631f2b1d64eb0 is 50, key is test_row_0/B:col10/1732416456846/Put/seqid=0 2024-11-24T02:47:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416518427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:38,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742163_1339 (size=12301) 2024-11-24T02:47:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T02:47:38,648 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:38,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-24T02:47:38,650 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:38,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:47:38,651 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:38,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:38,656 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-24T02:47:38,656 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:38,656 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,656 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:38,657 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-11-24T02:47:38,657 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-24T02:47:38,657 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:38,657 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,657 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:38,657 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,657 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,657 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:38,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:38,657 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-11-24T02:47:38,657 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:38,657 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,657 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:38,659 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-24T02:47:38,659 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:38,659 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,659 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:38,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:38,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416518736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:38,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:47:38,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T02:47:38,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:38,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:38,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:38,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:38,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:38,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/e6ffe64ba8c340bf804631f2b1d64eb0 2024-11-24T02:47:38,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/20d9058d3e6a447faa4b0f32f54bfb7d is 50, key is test_row_0/C:col10/1732416456846/Put/seqid=0 2024-11-24T02:47:38,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742164_1340 (size=12301) 2024-11-24T02:47:38,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T02:47:38,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:38,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:38,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:38,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:38,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:47:39,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T02:47:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:39,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:39,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47948 deadline: 1732416519247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:47:39,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/20d9058d3e6a447faa4b0f32f54bfb7d 2024-11-24T02:47:39,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T02:47:39,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/caa4ec3fc2e94eec998425aa4d508d79 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/caa4ec3fc2e94eec998425aa4d508d79 2024-11-24T02:47:39,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:39,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:39,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:39,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/caa4ec3fc2e94eec998425aa4d508d79, entries=200, sequenceid=446, filesize=14.4 K 2024-11-24T02:47:39,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/e6ffe64ba8c340bf804631f2b1d64eb0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/e6ffe64ba8c340bf804631f2b1d64eb0 2024-11-24T02:47:39,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/e6ffe64ba8c340bf804631f2b1d64eb0, entries=150, sequenceid=446, filesize=12.0 K 2024-11-24T02:47:39,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/20d9058d3e6a447faa4b0f32f54bfb7d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/20d9058d3e6a447faa4b0f32f54bfb7d 2024-11-24T02:47:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:47:39,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/20d9058d3e6a447faa4b0f32f54bfb7d, entries=150, sequenceid=446, filesize=12.0 K 2024-11-24T02:47:39,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a9560919f24867d41b2f73bc8d1a4195 in 1279ms, sequenceid=446, compaction requested=true 2024-11-24T02:47:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:39,279 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-24T02:47:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:39,279 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-24T02:47:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9560919f24867d41b2f73bc8d1a4195:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:39,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:39,281 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74658 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-24T02:47:39,281 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/B is initiating minor compaction (all files) 2024-11-24T02:47:39,281 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/B in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,281 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b763d43461a645838ebb6151442736df, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2b25f8f94dd04e118b58e5703b47994a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2016b7b8f9b84fbca8ec1270d304514c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/46bdc9c8c4b5449d8dffd980a770eece, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/f685adaea6854ae5a8f6ba6f543644cd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/e6ffe64ba8c340bf804631f2b1d64eb0] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=72.9 K 2024-11-24T02:47:39,282 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 79538 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-24T02:47:39,282 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/A is initiating minor compaction (all files) 2024-11-24T02:47:39,282 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/A in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,282 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ec4ae13e917144479517ef173e1b310b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/447e686fc070406ca0bb5b40622de7e2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f3cb58712bb494d848b00de5964533c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b0a6625297a341f19f72c0a57e1ca6a7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/457a3205f96c464ea3ff65bcdc3077eb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/caa4ec3fc2e94eec998425aa4d508d79] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=77.7 K 2024-11-24T02:47:39,282 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b763d43461a645838ebb6151442736df, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416452703 2024-11-24T02:47:39,282 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec4ae13e917144479517ef173e1b310b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416452703 2024-11-24T02:47:39,282 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b25f8f94dd04e118b58e5703b47994a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732416453958 2024-11-24T02:47:39,282 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447e686fc070406ca0bb5b40622de7e2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732416453958 2024-11-24T02:47:39,283 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f3cb58712bb494d848b00de5964533c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732416454260 2024-11-24T02:47:39,283 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2016b7b8f9b84fbca8ec1270d304514c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732416454260 2024-11-24T02:47:39,283 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0a6625297a341f19f72c0a57e1ca6a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416454414 2024-11-24T02:47:39,283 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 46bdc9c8c4b5449d8dffd980a770eece, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416454414 2024-11-24T02:47:39,283 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 457a3205f96c464ea3ff65bcdc3077eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732416455661 2024-11-24T02:47:39,283 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f685adaea6854ae5a8f6ba6f543644cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732416455661 2024-11-24T02:47:39,284 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa4ec3fc2e94eec998425aa4d508d79, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732416456821 2024-11-24T02:47:39,284 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ffe64ba8c340bf804631f2b1d64eb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732416456821 2024-11-24T02:47:39,302 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#B#compaction#294 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:39,302 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#A#compaction#293 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:39,302 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/09b6d48f37c4456284d8d0c0519a9c6c is 50, key is test_row_0/A:col10/1732416456846/Put/seqid=0 2024-11-24T02:47:39,302 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/69335fc57c9a40c48a513464105cdbf0 is 50, key is test_row_0/B:col10/1732416456846/Put/seqid=0 2024-11-24T02:47:39,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742165_1341 (size=13357) 2024-11-24T02:47:39,321 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/09b6d48f37c4456284d8d0c0519a9c6c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/09b6d48f37c4456284d8d0c0519a9c6c 2024-11-24T02:47:39,328 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/A of a9560919f24867d41b2f73bc8d1a4195 into 09b6d48f37c4456284d8d0c0519a9c6c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:39,328 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:39,328 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/A, priority=10, startTime=1732416459279; duration=0sec 2024-11-24T02:47:39,328 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:39,328 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:A 2024-11-24T02:47:39,328 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-24T02:47:39,330 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74658 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-24T02:47:39,331 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): a9560919f24867d41b2f73bc8d1a4195/C is initiating minor compaction (all files) 2024-11-24T02:47:39,331 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9560919f24867d41b2f73bc8d1a4195/C in TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,331 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/643d9458a232430c918f69b0897e767f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/e9a3b4657a8e4b44b93bd027ff1a05b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0fc6fbd426f149d187fcf03a60bef21c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/172877a0e8fe402b9b98acb285b3893a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a84ae41af7bd4385bf19cc47c8f8b099, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/20d9058d3e6a447faa4b0f32f54bfb7d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp, totalSize=72.9 K 2024-11-24T02:47:39,331 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 643d9458a232430c918f69b0897e767f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416452703 2024-11-24T02:47:39,332 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9a3b4657a8e4b44b93bd027ff1a05b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732416453958 2024-11-24T02:47:39,332 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc6fbd426f149d187fcf03a60bef21c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732416454260 2024-11-24T02:47:39,333 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 172877a0e8fe402b9b98acb285b3893a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732416454414 2024-11-24T02:47:39,333 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a84ae41af7bd4385bf19cc47c8f8b099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1732416455661 2024-11-24T02:47:39,333 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d9058d3e6a447faa4b0f32f54bfb7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732416456821 2024-11-24T02:47:39,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742166_1342 (size=13357) 2024-11-24T02:47:39,348 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9560919f24867d41b2f73bc8d1a4195#C#compaction#295 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:39,348 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/b38f42874a7440ba929f4fb651408605 is 50, key is test_row_0/C:col10/1732416456846/Put/seqid=0 2024-11-24T02:47:39,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742167_1343 (size=13357) 2024-11-24T02:47:39,368 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/b38f42874a7440ba929f4fb651408605 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b38f42874a7440ba929f4fb651408605 2024-11-24T02:47:39,377 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/C of a9560919f24867d41b2f73bc8d1a4195 into b38f42874a7440ba929f4fb651408605(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:39,377 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:39,377 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/C, priority=10, startTime=1732416459279; duration=0sec 2024-11-24T02:47:39,377 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:39,377 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:C 2024-11-24T02:47:39,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T02:47:39,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:39,418 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:47:39,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/df2c65686d0c4e3f99d2f64cfae14037 is 50, key is test_row_0/A:col10/1732416458093/Put/seqid=0 2024-11-24T02:47:39,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742168_1344 (size=12301) 2024-11-24T02:47:39,753 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/69335fc57c9a40c48a513464105cdbf0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/69335fc57c9a40c48a513464105cdbf0 2024-11-24T02:47:39,758 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in a9560919f24867d41b2f73bc8d1a4195/B of a9560919f24867d41b2f73bc8d1a4195 into 69335fc57c9a40c48a513464105cdbf0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:39,758 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:39,758 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., storeName=a9560919f24867d41b2f73bc8d1a4195/B, priority=10, startTime=1732416459279; duration=0sec 2024-11-24T02:47:39,758 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:39,758 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9560919f24867d41b2f73bc8d1a4195:B 2024-11-24T02:47:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:47:39,826 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/df2c65686d0c4e3f99d2f64cfae14037 2024-11-24T02:47:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/910ce1d5680a41079d477e475db8a0fb is 50, key is test_row_0/B:col10/1732416458093/Put/seqid=0 2024-11-24T02:47:39,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742169_1345 (size=12301) 2024-11-24T02:47:39,843 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/910ce1d5680a41079d477e475db8a0fb 2024-11-24T02:47:39,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/691bf4155ced474b9eac5e34d9d9bbf4 is 50, key is test_row_0/C:col10/1732416458093/Put/seqid=0 2024-11-24T02:47:39,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742170_1346 (size=12301) 2024-11-24T02:47:40,258 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/691bf4155ced474b9eac5e34d9d9bbf4 2024-11-24T02:47:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:40,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. as already flushing 2024-11-24T02:47:40,259 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-24T02:47:40,259 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:40,259 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,259 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/df2c65686d0c4e3f99d2f64cfae14037 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/df2c65686d0c4e3f99d2f64cfae14037 2024-11-24T02:47:40,269 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/df2c65686d0c4e3f99d2f64cfae14037, entries=150, sequenceid=472, filesize=12.0 K 2024-11-24T02:47:40,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/910ce1d5680a41079d477e475db8a0fb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/910ce1d5680a41079d477e475db8a0fb 2024-11-24T02:47:40,275 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/910ce1d5680a41079d477e475db8a0fb, entries=150, sequenceid=472, filesize=12.0 K 2024-11-24T02:47:40,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/691bf4155ced474b9eac5e34d9d9bbf4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/691bf4155ced474b9eac5e34d9d9bbf4 2024-11-24T02:47:40,281 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/691bf4155ced474b9eac5e34d9d9bbf4, entries=150, sequenceid=472, filesize=12.0 K 2024-11-24T02:47:40,282 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for a9560919f24867d41b2f73bc8d1a4195 in 864ms, sequenceid=472, compaction requested=false 2024-11-24T02:47:40,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for a9560919f24867d41b2f73bc8d1a4195: 2024-11-24T02:47:40,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:40,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-24T02:47:40,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-24T02:47:40,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-24T02:47:40,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6320 sec 2024-11-24T02:47:40,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.6390 sec 2024-11-24T02:47:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T02:47:40,796 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:40,898 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-24T02:47:40,898 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:40,898 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,898 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:40,908 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-24T02:47:40,908 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-24T02:47:40,908 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:40,908 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:40,908 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,908 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,908 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,908 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:40,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:40,929 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-24T02:47:40,929 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:47:40,929 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,929 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,929 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3339 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3333 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3281 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3362 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3325 2024-11-24T02:47:40,929 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:47:40,929 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:47:40,930 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:47:40,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:40,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:47:40,930 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:47:40,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T02:47:40,934 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416460934"}]},"ts":"1732416460934"} 2024-11-24T02:47:40,936 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:47:40,936 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:47:40,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:47:40,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, UNASSIGN}] 2024-11-24T02:47:40,940 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, UNASSIGN 2024-11-24T02:47:40,941 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a9560919f24867d41b2f73bc8d1a4195, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:40,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, UNASSIGN because future has completed 2024-11-24T02:47:40,944 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:47:40,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:47:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T02:47:41,096 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:41,096 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:47:41,096 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing a9560919f24867d41b2f73bc8d1a4195, disabling compactions & flushes 2024-11-24T02:47:41,096 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:41,096 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:41,096 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. after waiting 0 ms 2024-11-24T02:47:41,096 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:41,097 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing a9560919f24867d41b2f73bc8d1a4195 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T02:47:41,103 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/75d7c9a8d38445d88cc218c465054e05 is 50, key is test_row_1/A:col10/1732416460927/Put/seqid=0 2024-11-24T02:47:41,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742171_1347 (size=9857) 2024-11-24T02:47:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T02:47:41,509 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/75d7c9a8d38445d88cc218c465054e05 2024-11-24T02:47:41,520 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/acfc0183d4074a1f91f16da75e142266 is 50, key is test_row_1/B:col10/1732416460927/Put/seqid=0 2024-11-24T02:47:41,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742172_1348 (size=9857) 2024-11-24T02:47:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T02:47:41,926 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/acfc0183d4074a1f91f16da75e142266 2024-11-24T02:47:41,940 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/ced06c6b34574a39984801a7e1da4953 is 50, key is test_row_1/C:col10/1732416460927/Put/seqid=0 2024-11-24T02:47:41,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742173_1349 (size=9857) 2024-11-24T02:47:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T02:47:42,345 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/ced06c6b34574a39984801a7e1da4953 2024-11-24T02:47:42,352 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/A/75d7c9a8d38445d88cc218c465054e05 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/75d7c9a8d38445d88cc218c465054e05 2024-11-24T02:47:42,357 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/75d7c9a8d38445d88cc218c465054e05, entries=100, sequenceid=481, filesize=9.6 K 2024-11-24T02:47:42,358 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/B/acfc0183d4074a1f91f16da75e142266 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/acfc0183d4074a1f91f16da75e142266 2024-11-24T02:47:42,362 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/acfc0183d4074a1f91f16da75e142266, entries=100, sequenceid=481, filesize=9.6 K 2024-11-24T02:47:42,362 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/.tmp/C/ced06c6b34574a39984801a7e1da4953 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ced06c6b34574a39984801a7e1da4953 2024-11-24T02:47:42,365 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ced06c6b34574a39984801a7e1da4953, entries=100, sequenceid=481, filesize=9.6 K 2024-11-24T02:47:42,366 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a9560919f24867d41b2f73bc8d1a4195 in 1269ms, sequenceid=481, compaction requested=true 2024-11-24T02:47:42,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/96a0652f9f0f42de93f63fa40582a127, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ad1384ecc64c40cc907a0c135da30604, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/01b98497e5e247f2803e2ec91e49e7ca, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/aebae64ba45745379863affecb0b504b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a38fd17f0d044c98823def17d821ed6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/4fdd5f0f1fa14a50bd7ce099cac7c93f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ee276d4e6c984d3f8fd4f666b9ef228d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/6a01f50e965e4147b308b2fe8868a37c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f89eda28c7aa43d787f7ad45a3c2aabe, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/533152c8e50245558f6791d11bfd8707, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/af4880159d4f4812a057b056385c1b2c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/18e8350a4c204bb3bbf6d8c9a1c227e0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a2fb76d862e45f0b47a7807751be3a0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/19e7fda9ab224ca58272bf144d35c5bb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/0231fa628cbc4303a9464472e0ef550a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f320251ba634474b878501dfcbdc7d4b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/bba992cb5f154de69f017fc662b0975d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/a1e96e0e4e854f14b637c4ea4961a00c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f0bdd55c5b4f49b5aeffbb9fda9605e9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/c5bb8445d1374042b7cda8ba7ca1f16d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f59b51023f304e98a0d6d295a2507b4d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f8e223aaebb4322892b7489fa4fffaf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/1e2b553bf710473cb3ca519050117c03, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ae02a541fd194c759f84f01749c776e7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b06e079c01614d50b0385a0066de833c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ec4ae13e917144479517ef173e1b310b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/447e686fc070406ca0bb5b40622de7e2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f3cb58712bb494d848b00de5964533c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b0a6625297a341f19f72c0a57e1ca6a7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/457a3205f96c464ea3ff65bcdc3077eb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/caa4ec3fc2e94eec998425aa4d508d79] to archive 2024-11-24T02:47:42,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:47:42,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/96a0652f9f0f42de93f63fa40582a127 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/96a0652f9f0f42de93f63fa40582a127 2024-11-24T02:47:42,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ad1384ecc64c40cc907a0c135da30604 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ad1384ecc64c40cc907a0c135da30604 2024-11-24T02:47:42,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/01b98497e5e247f2803e2ec91e49e7ca to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/01b98497e5e247f2803e2ec91e49e7ca 2024-11-24T02:47:42,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/aebae64ba45745379863affecb0b504b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/aebae64ba45745379863affecb0b504b 2024-11-24T02:47:42,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a38fd17f0d044c98823def17d821ed6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a38fd17f0d044c98823def17d821ed6 2024-11-24T02:47:42,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/4fdd5f0f1fa14a50bd7ce099cac7c93f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/4fdd5f0f1fa14a50bd7ce099cac7c93f 2024-11-24T02:47:42,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ee276d4e6c984d3f8fd4f666b9ef228d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ee276d4e6c984d3f8fd4f666b9ef228d 2024-11-24T02:47:42,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/6a01f50e965e4147b308b2fe8868a37c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/6a01f50e965e4147b308b2fe8868a37c 2024-11-24T02:47:42,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f89eda28c7aa43d787f7ad45a3c2aabe to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f89eda28c7aa43d787f7ad45a3c2aabe 2024-11-24T02:47:42,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/533152c8e50245558f6791d11bfd8707 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/533152c8e50245558f6791d11bfd8707 2024-11-24T02:47:42,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/af4880159d4f4812a057b056385c1b2c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/af4880159d4f4812a057b056385c1b2c 2024-11-24T02:47:42,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/18e8350a4c204bb3bbf6d8c9a1c227e0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/18e8350a4c204bb3bbf6d8c9a1c227e0 2024-11-24T02:47:42,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a2fb76d862e45f0b47a7807751be3a0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/2a2fb76d862e45f0b47a7807751be3a0 2024-11-24T02:47:42,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/19e7fda9ab224ca58272bf144d35c5bb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/19e7fda9ab224ca58272bf144d35c5bb 2024-11-24T02:47:42,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/0231fa628cbc4303a9464472e0ef550a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/0231fa628cbc4303a9464472e0ef550a 2024-11-24T02:47:42,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f320251ba634474b878501dfcbdc7d4b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f320251ba634474b878501dfcbdc7d4b 2024-11-24T02:47:42,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/bba992cb5f154de69f017fc662b0975d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/bba992cb5f154de69f017fc662b0975d 2024-11-24T02:47:42,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/a1e96e0e4e854f14b637c4ea4961a00c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/a1e96e0e4e854f14b637c4ea4961a00c 2024-11-24T02:47:42,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f0bdd55c5b4f49b5aeffbb9fda9605e9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f0bdd55c5b4f49b5aeffbb9fda9605e9 2024-11-24T02:47:42,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/c5bb8445d1374042b7cda8ba7ca1f16d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/c5bb8445d1374042b7cda8ba7ca1f16d 2024-11-24T02:47:42,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f59b51023f304e98a0d6d295a2507b4d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/f59b51023f304e98a0d6d295a2507b4d 2024-11-24T02:47:42,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f8e223aaebb4322892b7489fa4fffaf to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f8e223aaebb4322892b7489fa4fffaf 2024-11-24T02:47:42,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/1e2b553bf710473cb3ca519050117c03 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/1e2b553bf710473cb3ca519050117c03 2024-11-24T02:47:42,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ae02a541fd194c759f84f01749c776e7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ae02a541fd194c759f84f01749c776e7 2024-11-24T02:47:42,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b06e079c01614d50b0385a0066de833c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b06e079c01614d50b0385a0066de833c 2024-11-24T02:47:42,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ec4ae13e917144479517ef173e1b310b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/ec4ae13e917144479517ef173e1b310b 2024-11-24T02:47:42,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/447e686fc070406ca0bb5b40622de7e2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/447e686fc070406ca0bb5b40622de7e2 2024-11-24T02:47:42,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f3cb58712bb494d848b00de5964533c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/7f3cb58712bb494d848b00de5964533c 2024-11-24T02:47:42,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b0a6625297a341f19f72c0a57e1ca6a7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/b0a6625297a341f19f72c0a57e1ca6a7 2024-11-24T02:47:42,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/457a3205f96c464ea3ff65bcdc3077eb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/457a3205f96c464ea3ff65bcdc3077eb 2024-11-24T02:47:42,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/caa4ec3fc2e94eec998425aa4d508d79 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/caa4ec3fc2e94eec998425aa4d508d79 2024-11-24T02:47:42,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/588c9c5654b2471785762adce2d710ea, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/db2291e5a6434a55a104d01c8e5e74ee, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/252b9f891be24bc9977b2a8207dc9bb9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/251af6a86472494ebdf4e11a572c5f9d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/fa55d8720e1d43b3a2d50c1ff0f72077, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2e739069a9634887af9fea70714c5bd9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/16194f941c5c4afa8c46af77e9807577, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/99aad9b4fb04425fa425c44ae238f672, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/255549aedf3f4dff9a5712d052585a4d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b1cf274adbe2479cb7584d26c42b8116, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/6b7c51ef376a464c819063b86182b779, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/1933a1be6aa24d19bb6d59200152fa99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b45e5777ef284ce6ae9347a3b1861fd0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/47910c94b7754be388ba42c115fc90b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/72e1a8fa4fbf45ff9038b10445d4f1e7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b889e0a2d2354f76a4330ba1f54948fc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/11e43d4be3ee4c69a0f2f473a2e4aeb5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/3f797d7498c54ce3889ff84ba3a494dc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/67f1405817fd45a3a93eae2b55cc139b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/4f9d489530584b09b2b36f4d51cfcec4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/0fd2de142f694b9a9778d4c8273d9d14, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/d1920b664cce44af9ccc9778960804d9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/ab6f33d8b7f24d878b5e5270820d784d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/132615f29d3140bdb8585720d74086f9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b763d43461a645838ebb6151442736df, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/dec27c94b00a44f28a8263cc713c5e03, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2b25f8f94dd04e118b58e5703b47994a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2016b7b8f9b84fbca8ec1270d304514c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/46bdc9c8c4b5449d8dffd980a770eece, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/f685adaea6854ae5a8f6ba6f543644cd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/e6ffe64ba8c340bf804631f2b1d64eb0] to archive 2024-11-24T02:47:42,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:47:42,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/588c9c5654b2471785762adce2d710ea to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/588c9c5654b2471785762adce2d710ea 2024-11-24T02:47:42,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/db2291e5a6434a55a104d01c8e5e74ee to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/db2291e5a6434a55a104d01c8e5e74ee 2024-11-24T02:47:42,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/252b9f891be24bc9977b2a8207dc9bb9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/252b9f891be24bc9977b2a8207dc9bb9 2024-11-24T02:47:42,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/251af6a86472494ebdf4e11a572c5f9d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/251af6a86472494ebdf4e11a572c5f9d 2024-11-24T02:47:42,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/fa55d8720e1d43b3a2d50c1ff0f72077 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/fa55d8720e1d43b3a2d50c1ff0f72077 2024-11-24T02:47:42,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2e739069a9634887af9fea70714c5bd9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2e739069a9634887af9fea70714c5bd9 2024-11-24T02:47:42,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/16194f941c5c4afa8c46af77e9807577 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/16194f941c5c4afa8c46af77e9807577 2024-11-24T02:47:42,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/99aad9b4fb04425fa425c44ae238f672 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/99aad9b4fb04425fa425c44ae238f672 2024-11-24T02:47:42,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/255549aedf3f4dff9a5712d052585a4d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/255549aedf3f4dff9a5712d052585a4d 2024-11-24T02:47:42,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b1cf274adbe2479cb7584d26c42b8116 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b1cf274adbe2479cb7584d26c42b8116 2024-11-24T02:47:42,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/6b7c51ef376a464c819063b86182b779 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/6b7c51ef376a464c819063b86182b779 2024-11-24T02:47:42,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/1933a1be6aa24d19bb6d59200152fa99 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/1933a1be6aa24d19bb6d59200152fa99 2024-11-24T02:47:42,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b45e5777ef284ce6ae9347a3b1861fd0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b45e5777ef284ce6ae9347a3b1861fd0 2024-11-24T02:47:42,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/47910c94b7754be388ba42c115fc90b7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/47910c94b7754be388ba42c115fc90b7 2024-11-24T02:47:42,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/72e1a8fa4fbf45ff9038b10445d4f1e7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/72e1a8fa4fbf45ff9038b10445d4f1e7 2024-11-24T02:47:42,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b889e0a2d2354f76a4330ba1f54948fc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b889e0a2d2354f76a4330ba1f54948fc 2024-11-24T02:47:42,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/11e43d4be3ee4c69a0f2f473a2e4aeb5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/11e43d4be3ee4c69a0f2f473a2e4aeb5 2024-11-24T02:47:42,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/3f797d7498c54ce3889ff84ba3a494dc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/3f797d7498c54ce3889ff84ba3a494dc 2024-11-24T02:47:42,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/67f1405817fd45a3a93eae2b55cc139b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/67f1405817fd45a3a93eae2b55cc139b 2024-11-24T02:47:42,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/4f9d489530584b09b2b36f4d51cfcec4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/4f9d489530584b09b2b36f4d51cfcec4 2024-11-24T02:47:42,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/0fd2de142f694b9a9778d4c8273d9d14 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/0fd2de142f694b9a9778d4c8273d9d14 2024-11-24T02:47:42,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/d1920b664cce44af9ccc9778960804d9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/d1920b664cce44af9ccc9778960804d9 2024-11-24T02:47:42,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/ab6f33d8b7f24d878b5e5270820d784d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/ab6f33d8b7f24d878b5e5270820d784d 2024-11-24T02:47:42,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/132615f29d3140bdb8585720d74086f9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/132615f29d3140bdb8585720d74086f9 2024-11-24T02:47:42,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b763d43461a645838ebb6151442736df to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/b763d43461a645838ebb6151442736df 2024-11-24T02:47:42,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/dec27c94b00a44f28a8263cc713c5e03 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/dec27c94b00a44f28a8263cc713c5e03 2024-11-24T02:47:42,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2b25f8f94dd04e118b58e5703b47994a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2b25f8f94dd04e118b58e5703b47994a 2024-11-24T02:47:42,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2016b7b8f9b84fbca8ec1270d304514c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/2016b7b8f9b84fbca8ec1270d304514c 2024-11-24T02:47:42,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/46bdc9c8c4b5449d8dffd980a770eece to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/46bdc9c8c4b5449d8dffd980a770eece 2024-11-24T02:47:42,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/f685adaea6854ae5a8f6ba6f543644cd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/f685adaea6854ae5a8f6ba6f543644cd 2024-11-24T02:47:42,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/e6ffe64ba8c340bf804631f2b1d64eb0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/e6ffe64ba8c340bf804631f2b1d64eb0 2024-11-24T02:47:42,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8387e0c7007c4c99bcd417e31c7585bf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/933c974d88484e789ad6119aacb1cd09, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/05c7d653566749b5b2d9d9e781f6096b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/07fca0e6133446e2aafc948e15022bce, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4a27ba4fb2a74ad9b7c9999d80977463, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b7fdc8d77ba942f7b1a35a6244783600, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3e2b68c79259496b8b5d5d741214c4b2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0959399b436f42e5a29a7392a41cf891, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6763711416b94fd087e10cedf60fd346, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8e3097ba7d1e479bbe882f821ef5d832, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/d77da9827b614ba4b71c657a80636439, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6b3ac041a51b4a718c33c9feffae6241, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/eb2093c383e84c49af7211b0ca9bf050, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/c3517dbb2d9e4095aeee651389be3cf8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/fea34f3ec1854fc28522e4833e19fa82, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/66318575b15b46d7ad7c984b443c0f02, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/7a1c7d262e644165827c85c249096b72, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4397ca8909c8496d991159a5e8f7648a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3f36a2603fe64f56975f05660815177b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a204c01a6c6c4132826aa4df6573b2a1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0f965c75a1884187af3f7a4912b1a6d9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ce763f14700645c0a0210eb2f79be2e0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/bc891a6034f9438c88a711cc98ff8ed1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/643d9458a232430c918f69b0897e767f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/39b8f096feb243aea34438d89ff6ed5b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/e9a3b4657a8e4b44b93bd027ff1a05b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0fc6fbd426f149d187fcf03a60bef21c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/172877a0e8fe402b9b98acb285b3893a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a84ae41af7bd4385bf19cc47c8f8b099, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/20d9058d3e6a447faa4b0f32f54bfb7d] to archive 2024-11-24T02:47:42,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:47:42,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8387e0c7007c4c99bcd417e31c7585bf to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8387e0c7007c4c99bcd417e31c7585bf 2024-11-24T02:47:42,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/933c974d88484e789ad6119aacb1cd09 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/933c974d88484e789ad6119aacb1cd09 2024-11-24T02:47:42,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/05c7d653566749b5b2d9d9e781f6096b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/05c7d653566749b5b2d9d9e781f6096b 2024-11-24T02:47:42,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/07fca0e6133446e2aafc948e15022bce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/07fca0e6133446e2aafc948e15022bce 2024-11-24T02:47:42,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4a27ba4fb2a74ad9b7c9999d80977463 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4a27ba4fb2a74ad9b7c9999d80977463 2024-11-24T02:47:42,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b7fdc8d77ba942f7b1a35a6244783600 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b7fdc8d77ba942f7b1a35a6244783600 2024-11-24T02:47:42,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3e2b68c79259496b8b5d5d741214c4b2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3e2b68c79259496b8b5d5d741214c4b2 2024-11-24T02:47:42,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0959399b436f42e5a29a7392a41cf891 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0959399b436f42e5a29a7392a41cf891 2024-11-24T02:47:42,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6763711416b94fd087e10cedf60fd346 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6763711416b94fd087e10cedf60fd346 2024-11-24T02:47:42,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8e3097ba7d1e479bbe882f821ef5d832 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/8e3097ba7d1e479bbe882f821ef5d832 2024-11-24T02:47:42,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/d77da9827b614ba4b71c657a80636439 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/d77da9827b614ba4b71c657a80636439 2024-11-24T02:47:42,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6b3ac041a51b4a718c33c9feffae6241 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/6b3ac041a51b4a718c33c9feffae6241 2024-11-24T02:47:42,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/eb2093c383e84c49af7211b0ca9bf050 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/eb2093c383e84c49af7211b0ca9bf050 2024-11-24T02:47:42,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/c3517dbb2d9e4095aeee651389be3cf8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/c3517dbb2d9e4095aeee651389be3cf8 2024-11-24T02:47:42,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/fea34f3ec1854fc28522e4833e19fa82 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/fea34f3ec1854fc28522e4833e19fa82 2024-11-24T02:47:42,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/66318575b15b46d7ad7c984b443c0f02 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/66318575b15b46d7ad7c984b443c0f02 2024-11-24T02:47:42,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/7a1c7d262e644165827c85c249096b72 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/7a1c7d262e644165827c85c249096b72 2024-11-24T02:47:42,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4397ca8909c8496d991159a5e8f7648a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/4397ca8909c8496d991159a5e8f7648a 2024-11-24T02:47:42,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3f36a2603fe64f56975f05660815177b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/3f36a2603fe64f56975f05660815177b 2024-11-24T02:47:42,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a204c01a6c6c4132826aa4df6573b2a1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a204c01a6c6c4132826aa4df6573b2a1 2024-11-24T02:47:42,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0f965c75a1884187af3f7a4912b1a6d9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0f965c75a1884187af3f7a4912b1a6d9 2024-11-24T02:47:42,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ce763f14700645c0a0210eb2f79be2e0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ce763f14700645c0a0210eb2f79be2e0 2024-11-24T02:47:42,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/bc891a6034f9438c88a711cc98ff8ed1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/bc891a6034f9438c88a711cc98ff8ed1 2024-11-24T02:47:42,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/16d7f0bfd8704b8fa22ef0e0bd2d53b7 2024-11-24T02:47:42,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/643d9458a232430c918f69b0897e767f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/643d9458a232430c918f69b0897e767f 2024-11-24T02:47:42,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/39b8f096feb243aea34438d89ff6ed5b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/39b8f096feb243aea34438d89ff6ed5b 2024-11-24T02:47:42,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/e9a3b4657a8e4b44b93bd027ff1a05b7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/e9a3b4657a8e4b44b93bd027ff1a05b7 2024-11-24T02:47:42,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0fc6fbd426f149d187fcf03a60bef21c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/0fc6fbd426f149d187fcf03a60bef21c 2024-11-24T02:47:42,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/172877a0e8fe402b9b98acb285b3893a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/172877a0e8fe402b9b98acb285b3893a 2024-11-24T02:47:42,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a84ae41af7bd4385bf19cc47c8f8b099 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/a84ae41af7bd4385bf19cc47c8f8b099 2024-11-24T02:47:42,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/20d9058d3e6a447faa4b0f32f54bfb7d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/20d9058d3e6a447faa4b0f32f54bfb7d 2024-11-24T02:47:42,455 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/recovered.edits/484.seqid, newMaxSeqId=484, maxSeqId=1 2024-11-24T02:47:42,455 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. 2024-11-24T02:47:42,455 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for a9560919f24867d41b2f73bc8d1a4195: Waiting for close lock at 1732416461096Running coprocessor pre-close hooks at 1732416461096Disabling compacts and flushes for region at 1732416461096Disabling writes for close at 1732416461096Obtaining lock to block concurrent updates at 1732416461097 (+1 ms)Preparing flush snapshotting stores in a9560919f24867d41b2f73bc8d1a4195 at 1732416461097Finished memstore snapshotting TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732416461097Flushing stores of TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195. at 1732416461098 (+1 ms)Flushing a9560919f24867d41b2f73bc8d1a4195/A: creating writer at 1732416461098Flushing a9560919f24867d41b2f73bc8d1a4195/A: appending metadata at 1732416461102 (+4 ms)Flushing a9560919f24867d41b2f73bc8d1a4195/A: closing flushed file at 1732416461102Flushing a9560919f24867d41b2f73bc8d1a4195/B: creating writer at 1732416461516 (+414 ms)Flushing a9560919f24867d41b2f73bc8d1a4195/B: appending metadata at 1732416461520 (+4 ms)Flushing a9560919f24867d41b2f73bc8d1a4195/B: closing flushed file at 1732416461520Flushing a9560919f24867d41b2f73bc8d1a4195/C: creating writer at 1732416461934 (+414 ms)Flushing a9560919f24867d41b2f73bc8d1a4195/C: appending metadata at 1732416461940 (+6 ms)Flushing a9560919f24867d41b2f73bc8d1a4195/C: closing flushed file at 1732416461940Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72b9c615: reopening flushed file at 1732416462351 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c406b64: reopening flushed file at 1732416462357 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb14bf3: reopening flushed file at 1732416462362 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a9560919f24867d41b2f73bc8d1a4195 in 1269ms, sequenceid=481, compaction requested=true at 1732416462366 (+4 ms)Writing region close event to WAL at 1732416462452 (+86 ms)Running coprocessor post-close hooks at 1732416462455 (+3 ms)Closed at 1732416462455 2024-11-24T02:47:42,457 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:42,457 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=a9560919f24867d41b2f73bc8d1a4195, regionState=CLOSED 2024-11-24T02:47:42,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:47:42,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-24T02:47:42,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure a9560919f24867d41b2f73bc8d1a4195, server=7c69a60bd8f6,39461,1732416369827 in 1.5150 sec 2024-11-24T02:47:42,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-24T02:47:42,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9560919f24867d41b2f73bc8d1a4195, UNASSIGN in 1.5210 sec 2024-11-24T02:47:42,462 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-24T02:47:42,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-24T02:47:42,463 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416462463"}]},"ts":"1732416462463"} 2024-11-24T02:47:42,465 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:47:42,465 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:47:42,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5340 sec 2024-11-24T02:47:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T02:47:43,077 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:43,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:47:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:43,079 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-24T02:47:43,080 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:43,083 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:43,089 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/recovered.edits] 2024-11-24T02:47:43,091 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/09b6d48f37c4456284d8d0c0519a9c6c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/09b6d48f37c4456284d8d0c0519a9c6c 2024-11-24T02:47:43,093 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/75d7c9a8d38445d88cc218c465054e05 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/75d7c9a8d38445d88cc218c465054e05 2024-11-24T02:47:43,094 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/df2c65686d0c4e3f99d2f64cfae14037 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/A/df2c65686d0c4e3f99d2f64cfae14037 2024-11-24T02:47:43,096 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/69335fc57c9a40c48a513464105cdbf0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/69335fc57c9a40c48a513464105cdbf0 2024-11-24T02:47:43,097 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/910ce1d5680a41079d477e475db8a0fb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/910ce1d5680a41079d477e475db8a0fb 2024-11-24T02:47:43,098 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/acfc0183d4074a1f91f16da75e142266 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/B/acfc0183d4074a1f91f16da75e142266 2024-11-24T02:47:43,099 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/691bf4155ced474b9eac5e34d9d9bbf4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/691bf4155ced474b9eac5e34d9d9bbf4 2024-11-24T02:47:43,100 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b38f42874a7440ba929f4fb651408605 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/b38f42874a7440ba929f4fb651408605 2024-11-24T02:47:43,101 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ced06c6b34574a39984801a7e1da4953 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/C/ced06c6b34574a39984801a7e1da4953 2024-11-24T02:47:43,103 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/recovered.edits/484.seqid to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195/recovered.edits/484.seqid 2024-11-24T02:47:43,104 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/a9560919f24867d41b2f73bc8d1a4195 2024-11-24T02:47:43,104 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:47:43,106 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:43,108 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:47:43,110 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:47:43,112 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:43,112 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:47:43,112 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416463112"}]},"ts":"9223372036854775807"} 2024-11-24T02:47:43,114 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:47:43,114 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a9560919f24867d41b2f73bc8d1a4195, NAME => 'TestAcidGuarantees,,1732416436446.a9560919f24867d41b2f73bc8d1a4195.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:47:43,114 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:47:43,114 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416463114"}]},"ts":"9223372036854775807"} 2024-11-24T02:47:43,116 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:47:43,117 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:43,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-24T02:47:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-24T02:47:43,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:47:43,187 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:43,199 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=697 (was 696) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10014 (was 10486) 2024-11-24T02:47:43,207 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=697, ProcessCount=11, AvailableMemoryMB=10014 2024-11-24T02:47:43,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:47:43,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:47:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:43,210 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:47:43,210 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:43,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-24T02:47:43,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T02:47:43,211 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:47:43,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742174_1350 (size=959) 2024-11-24T02:47:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T02:47:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T02:47:43,618 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:47:43,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742175_1351 (size=53) 2024-11-24T02:47:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T02:47:44,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:47:44,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 859cf8a736ca56969dfab9f223200140, disabling compactions & flushes 2024-11-24T02:47:44,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:44,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:44,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. after waiting 0 ms 2024-11-24T02:47:44,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:44,024 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:44,024 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 859cf8a736ca56969dfab9f223200140: Waiting for close lock at 1732416464024Disabling compacts and flushes for region at 1732416464024Disabling writes for close at 1732416464024Writing region close event to WAL at 1732416464024Closed at 1732416464024 2024-11-24T02:47:44,026 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:47:44,026 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416464026"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416464026"}]},"ts":"1732416464026"} 2024-11-24T02:47:44,028 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:47:44,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:47:44,030 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416464030"}]},"ts":"1732416464030"} 2024-11-24T02:47:44,033 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:47:44,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, ASSIGN}] 2024-11-24T02:47:44,035 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, ASSIGN 2024-11-24T02:47:44,036 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=false 2024-11-24T02:47:44,187 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:44,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, ASSIGN because future has completed 2024-11-24T02:47:44,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:47:44,346 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:44,347 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:47:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T02:47:44,347 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,347 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:47:44,347 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,348 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,350 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,352 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 859cf8a736ca56969dfab9f223200140 columnFamilyName A 2024-11-24T02:47:44,352 DEBUG [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:44,353 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(327): Store=859cf8a736ca56969dfab9f223200140/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:44,353 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,354 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 859cf8a736ca56969dfab9f223200140 columnFamilyName B 2024-11-24T02:47:44,354 DEBUG [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:44,354 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(327): Store=859cf8a736ca56969dfab9f223200140/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:44,354 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,355 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 859cf8a736ca56969dfab9f223200140 columnFamilyName C 2024-11-24T02:47:44,356 DEBUG [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:44,356 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(327): Store=859cf8a736ca56969dfab9f223200140/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:44,356 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,357 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,357 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,358 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,358 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,359 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:47:44,360 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,362 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:47:44,362 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 859cf8a736ca56969dfab9f223200140; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69078817, jitterRate=0.029354587197303772}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:47:44,362 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:44,363 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 859cf8a736ca56969dfab9f223200140: Running coprocessor pre-open hook at 1732416464348Writing region info on filesystem at 1732416464348Initializing all the Stores at 1732416464349 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416464349Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416464350 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416464350Cleaning up temporary data from old regions at 1732416464358 (+8 ms)Running coprocessor post-open hooks at 1732416464362 (+4 ms)Region opened successfully at 1732416464363 (+1 ms) 2024-11-24T02:47:44,363 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., pid=93, masterSystemTime=1732416464341 2024-11-24T02:47:44,365 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:44,365 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:44,366 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:44,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:47:44,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-24T02:47:44,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 in 180 msec 2024-11-24T02:47:44,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-24T02:47:44,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, ASSIGN in 338 msec 2024-11-24T02:47:44,374 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:47:44,374 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416464374"}]},"ts":"1732416464374"} 2024-11-24T02:47:44,375 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:47:44,376 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:47:44,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-24T02:47:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-24T02:47:45,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:45,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:45,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:45,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:45,359 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:45,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c25519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:45,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:45,361 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634649ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:45,362 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:45,363 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:45,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:45,364 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:47:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,43749,1732416369043 2024-11-24T02:47:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51d18b15 2024-11-24T02:47:45,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:47:45,366 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:47:45,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:47:45,367 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:47:45,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T02:47:45,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742176_1352 (size=995) 2024-11-24T02:47:45,780 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-24T02:47:45,780 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-24T02:47:45,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:47:45,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, REOPEN/MOVE}] 2024-11-24T02:47:45,786 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, REOPEN/MOVE 2024-11-24T02:47:45,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:45,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, REOPEN/MOVE because future has completed 2024-11-24T02:47:45,788 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:47:45,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:47:45,941 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:45,941 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:47:45,941 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 859cf8a736ca56969dfab9f223200140, disabling compactions & flushes 2024-11-24T02:47:45,941 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:45,941 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:45,941 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. after waiting 0 ms 2024-11-24T02:47:45,941 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:45,945 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T02:47:45,945 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:45,946 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 859cf8a736ca56969dfab9f223200140: Waiting for close lock at 1732416465941Running coprocessor pre-close hooks at 1732416465941Disabling compacts and flushes for region at 1732416465941Disabling writes for close at 1732416465941Writing region close event to WAL at 1732416465941Running coprocessor post-close hooks at 1732416465945 (+4 ms)Closed at 1732416465945 2024-11-24T02:47:45,946 WARN [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 859cf8a736ca56969dfab9f223200140 to self. 2024-11-24T02:47:45,947 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:45,948 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=CLOSED 2024-11-24T02:47:45,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:47:45,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-24T02:47:45,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 in 162 msec 2024-11-24T02:47:45,953 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, REOPEN/MOVE; state=CLOSED, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=true 2024-11-24T02:47:46,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, REOPEN/MOVE because future has completed 2024-11-24T02:47:46,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:47:46,259 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,259 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:47:46,260 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,260 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:47:46,260 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,260 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,262 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,263 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 859cf8a736ca56969dfab9f223200140 columnFamilyName A 2024-11-24T02:47:46,264 DEBUG [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:46,265 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(327): Store=859cf8a736ca56969dfab9f223200140/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:46,265 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,266 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 859cf8a736ca56969dfab9f223200140 columnFamilyName B 2024-11-24T02:47:46,266 DEBUG [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:46,266 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(327): Store=859cf8a736ca56969dfab9f223200140/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:46,266 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,267 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 859cf8a736ca56969dfab9f223200140 columnFamilyName C 2024-11-24T02:47:46,267 DEBUG [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:46,268 INFO [StoreOpener-859cf8a736ca56969dfab9f223200140-1 {}] regionserver.HStore(327): Store=859cf8a736ca56969dfab9f223200140/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:47:46,268 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,268 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,269 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,270 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,270 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,271 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:47:46,272 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,273 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 859cf8a736ca56969dfab9f223200140; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64191485, jitterRate=-0.04347233474254608}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:47:46,273 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,273 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 859cf8a736ca56969dfab9f223200140: Running coprocessor pre-open hook at 1732416466260Writing region info on filesystem at 1732416466260Initializing all the Stores at 1732416466261 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416466261Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416466262 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416466262Cleaning up temporary data from old regions at 1732416466270 (+8 ms)Running coprocessor post-open hooks at 1732416466273 (+3 ms)Region opened successfully at 1732416466273 2024-11-24T02:47:46,274 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., pid=98, masterSystemTime=1732416466256 2024-11-24T02:47:46,276 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,276 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,276 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=OPEN, openSeqNum=5, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:47:46,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-24T02:47:46,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 in 173 msec 2024-11-24T02:47:46,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-24T02:47:46,282 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, REOPEN/MOVE in 495 msec 2024-11-24T02:47:46,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-24T02:47:46,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-24T02:47:46,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-11-24T02:47:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-24T02:47:46,288 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:46,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2d70d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582da546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5e598e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f62a2a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,299 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55fe0063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,301 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ca67be9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,305 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443dd1f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467074d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@702bd4dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,311 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40044ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482a749e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,323 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54298b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,325 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c815554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,327 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5251e714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,332 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,333 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d53467a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee20608, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,337 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710d9178, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,339 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf64751, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:47:46,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:47:46,341 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:47:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:47:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:47:46,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c99359d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:47:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:47:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:47:46,343 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:47:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30724328, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:47:46,344 DEBUG [Thread-1614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,348 DEBUG [Thread-1616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-24T02:47:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:46,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:46,352 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:46,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,356 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,357 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,358 DEBUG [Thread-1601 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,359 DEBUG [Thread-1601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,359 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,380 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,389 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,392 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:46,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:46,393 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,395 DEBUG [Thread-1603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,395 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,405 DEBUG [Thread-1612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:47:46,406 DEBUG [Thread-1612 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:47:46,406 DEBUG [Thread-1612 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:47:46,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:47:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:47:46,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112451325bbcc3d947ef8850fdfe8f592f35_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416466385/Put/seqid=0 2024-11-24T02:47:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416526441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416526443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416526458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416526463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416526463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742177_1353 (size=14594) 2024-11-24T02:47:46,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:46,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112451325bbcc3d947ef8850fdfe8f592f35_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112451325bbcc3d947ef8850fdfe8f592f35_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:46,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/51d977a6bf614a0e923d6031d57b61ff, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:46,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/51d977a6bf614a0e923d6031d57b61ff is 175, key is test_row_0/A:col10/1732416466385/Put/seqid=0 2024-11-24T02:47:46,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:46,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742178_1354 (size=39549) 2024-11-24T02:47:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416526566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416526566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416526577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416526586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416526586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:46,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:46,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:46,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416526779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416526779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416526798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416526796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416526798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:46,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:46,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/51d977a6bf614a0e923d6031d57b61ff 2024-11-24T02:47:46,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:46,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f5c9467be7f643eb90cc2222f6aef4eb is 50, key is test_row_0/B:col10/1732416466385/Put/seqid=0 2024-11-24T02:47:46,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:46,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:47,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742179_1355 (size=12001) 2024-11-24T02:47:47,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f5c9467be7f643eb90cc2222f6aef4eb 2024-11-24T02:47:47,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/3737dabd772941d2a1b065154ef87091 is 50, key is test_row_0/C:col10/1732416466385/Put/seqid=0 2024-11-24T02:47:47,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742180_1356 (size=12001) 2024-11-24T02:47:47,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/3737dabd772941d2a1b065154ef87091 2024-11-24T02:47:47,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/51d977a6bf614a0e923d6031d57b61ff as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/51d977a6bf614a0e923d6031d57b61ff 2024-11-24T02:47:47,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/51d977a6bf614a0e923d6031d57b61ff, entries=200, sequenceid=15, filesize=38.6 K 2024-11-24T02:47:47,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f5c9467be7f643eb90cc2222f6aef4eb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f5c9467be7f643eb90cc2222f6aef4eb 2024-11-24T02:47:47,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f5c9467be7f643eb90cc2222f6aef4eb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T02:47:47,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/3737dabd772941d2a1b065154ef87091 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/3737dabd772941d2a1b065154ef87091 2024-11-24T02:47:47,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/3737dabd772941d2a1b065154ef87091, entries=150, sequenceid=15, filesize=11.7 K 2024-11-24T02:47:47,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 859cf8a736ca56969dfab9f223200140 in 691ms, sequenceid=15, compaction requested=false 2024-11-24T02:47:47,084 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T02:47:47,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:47,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:47,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T02:47:47,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d34c9be9ea324e9db4debee6586cffe6_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416467091/Put/seqid=0 2024-11-24T02:47:47,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742181_1357 (size=17034) 2024-11-24T02:47:47,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:47,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:47,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416527117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416527120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,131 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d34c9be9ea324e9db4debee6586cffe6_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d34c9be9ea324e9db4debee6586cffe6_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:47,132 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/4ea3ffdc3d8f415fa4c28d7194d869b0, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:47,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/4ea3ffdc3d8f415fa4c28d7194d869b0 is 175, key is test_row_0/A:col10/1732416467091/Put/seqid=0 2024-11-24T02:47:47,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416527124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416527125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416527126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742182_1358 (size=48139) 2024-11-24T02:47:47,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416527237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416527247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416527247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416527247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:47,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:47,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:47,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:47,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416527447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416527459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416527459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416527467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:47,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/4ea3ffdc3d8f415fa4c28d7194d869b0 2024-11-24T02:47:47,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/2e546009c70246d1913ddbf1176f1225 is 50, key is test_row_0/B:col10/1732416467091/Put/seqid=0 2024-11-24T02:47:47,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:47,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742183_1359 (size=12001) 2024-11-24T02:47:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/2e546009c70246d1913ddbf1176f1225 2024-11-24T02:47:47,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/13c1a2990ed44ddfbfc2b320a4a86d09 is 50, key is test_row_0/C:col10/1732416467091/Put/seqid=0 2024-11-24T02:47:47,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742184_1360 (size=12001) 2024-11-24T02:47:47,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/13c1a2990ed44ddfbfc2b320a4a86d09 2024-11-24T02:47:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/4ea3ffdc3d8f415fa4c28d7194d869b0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/4ea3ffdc3d8f415fa4c28d7194d869b0 2024-11-24T02:47:47,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/4ea3ffdc3d8f415fa4c28d7194d869b0, entries=250, sequenceid=42, filesize=47.0 K 2024-11-24T02:47:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/2e546009c70246d1913ddbf1176f1225 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2e546009c70246d1913ddbf1176f1225 2024-11-24T02:47:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2e546009c70246d1913ddbf1176f1225, entries=150, sequenceid=42, filesize=11.7 K 2024-11-24T02:47:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/13c1a2990ed44ddfbfc2b320a4a86d09 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/13c1a2990ed44ddfbfc2b320a4a86d09 2024-11-24T02:47:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/13c1a2990ed44ddfbfc2b320a4a86d09, entries=150, sequenceid=42, filesize=11.7 K 2024-11-24T02:47:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 859cf8a736ca56969dfab9f223200140 in 542ms, sequenceid=42, compaction requested=false 2024-11-24T02:47:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:47,693 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:47,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b12f4457ea754559b038b09fce94a01e_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416467664/Put/seqid=0 2024-11-24T02:47:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742185_1361 (size=12154) 2024-11-24T02:47:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:47,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:47,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:47,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:47,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:47,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416527889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416527889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416527890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416527890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:47,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416527891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:47,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416528006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416528006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416528006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416528007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416528007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,042 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:47:48,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:48,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:48,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,116 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:48,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b12f4457ea754559b038b09fce94a01e_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b12f4457ea754559b038b09fce94a01e_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:48,120 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/799ebc74ec254b0b83522adc0967f820, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:48,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/799ebc74ec254b0b83522adc0967f820 is 175, key is test_row_0/A:col10/1732416467664/Put/seqid=0 2024-11-24T02:47:48,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742186_1362 (size=30951) 2024-11-24T02:47:48,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416528217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416528217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416528217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416528217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416528226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:48,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:48,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:48,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:48,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:48,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,531 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/799ebc74ec254b0b83522adc0967f820 2024-11-24T02:47:48,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416528526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/75948c660fc64a6097e72cd8ca8eb019 is 50, key is test_row_0/B:col10/1732416467664/Put/seqid=0 2024-11-24T02:47:48,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742187_1363 (size=9657) 2024-11-24T02:47:48,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416528537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416528537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416528537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416528546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:48,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/75948c660fc64a6097e72cd8ca8eb019 2024-11-24T02:47:48,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:48,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:48,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:48,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:48,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/4d861d44c6dd496f9220deb2caf24702 is 50, key is test_row_0/C:col10/1732416467664/Put/seqid=0 2024-11-24T02:47:48,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742188_1364 (size=9657) 2024-11-24T02:47:48,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/4d861d44c6dd496f9220deb2caf24702 2024-11-24T02:47:48,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/799ebc74ec254b0b83522adc0967f820 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/799ebc74ec254b0b83522adc0967f820 2024-11-24T02:47:48,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/799ebc74ec254b0b83522adc0967f820, entries=150, sequenceid=53, filesize=30.2 K 2024-11-24T02:47:49,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/75948c660fc64a6097e72cd8ca8eb019 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/75948c660fc64a6097e72cd8ca8eb019 2024-11-24T02:47:49,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/75948c660fc64a6097e72cd8ca8eb019, entries=100, sequenceid=53, filesize=9.4 K 2024-11-24T02:47:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/4d861d44c6dd496f9220deb2caf24702 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4d861d44c6dd496f9220deb2caf24702 2024-11-24T02:47:49,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4d861d44c6dd496f9220deb2caf24702, entries=100, sequenceid=53, filesize=9.4 K 2024-11-24T02:47:49,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 859cf8a736ca56969dfab9f223200140 in 1339ms, sequenceid=53, compaction requested=true 2024-11-24T02:47:49,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:49,033 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:49,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:49,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:49,033 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:49,034 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/A is initiating minor compaction (all files) 2024-11-24T02:47:49,034 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/A in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,034 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/51d977a6bf614a0e923d6031d57b61ff, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/4ea3ffdc3d8f415fa4c28d7194d869b0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/799ebc74ec254b0b83522adc0967f820] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=115.9 K 2024-11-24T02:47:49,034 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,034 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/51d977a6bf614a0e923d6031d57b61ff, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/4ea3ffdc3d8f415fa4c28d7194d869b0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/799ebc74ec254b0b83522adc0967f820] 2024-11-24T02:47:49,034 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:49,034 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d977a6bf614a0e923d6031d57b61ff, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416466369 2024-11-24T02:47:49,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:49,035 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea3ffdc3d8f415fa4c28d7194d869b0, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732416466441 2024-11-24T02:47:49,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:49,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:49,037 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 799ebc74ec254b0b83522adc0967f820, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732416467121 2024-11-24T02:47:49,038 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:49,038 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/B is initiating minor compaction (all files) 2024-11-24T02:47:49,038 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/B in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,038 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f5c9467be7f643eb90cc2222f6aef4eb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2e546009c70246d1913ddbf1176f1225, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/75948c660fc64a6097e72cd8ca8eb019] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=32.9 K 2024-11-24T02:47:49,038 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c9467be7f643eb90cc2222f6aef4eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416466369 2024-11-24T02:47:49,039 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e546009c70246d1913ddbf1176f1225, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732416466441 2024-11-24T02:47:49,039 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75948c660fc64a6097e72cd8ca8eb019, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732416467664 2024-11-24T02:47:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:49,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:49,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T02:47:49,055 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:49,062 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:49,062 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/297e9098f6c64246860e8f55dcb9307d is 50, key is test_row_0/B:col10/1732416467664/Put/seqid=0 2024-11-24T02:47:49,073 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411248132bc1f86d14fe38f2d31f99098709c_859cf8a736ca56969dfab9f223200140 store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:49,077 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411248132bc1f86d14fe38f2d31f99098709c_859cf8a736ca56969dfab9f223200140, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:49,077 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248132bc1f86d14fe38f2d31f99098709c_859cf8a736ca56969dfab9f223200140 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:49,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416529068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416529069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416529078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416529079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416529079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742189_1365 (size=12104) 2024-11-24T02:47:49,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b8dc11a2397f45e08a651e9207a1e3b9_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416469040/Put/seqid=0 2024-11-24T02:47:49,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742190_1366 (size=4469) 2024-11-24T02:47:49,106 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/297e9098f6c64246860e8f55dcb9307d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/297e9098f6c64246860e8f55dcb9307d 2024-11-24T02:47:49,106 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#A#compaction#311 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:49,107 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/036b74ef82504473a23fcb075de9b103 is 175, key is test_row_0/A:col10/1732416467664/Put/seqid=0 2024-11-24T02:47:49,113 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 859cf8a736ca56969dfab9f223200140/B of 859cf8a736ca56969dfab9f223200140 into 297e9098f6c64246860e8f55dcb9307d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:49,113 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:49,113 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/B, priority=13, startTime=1732416469034; duration=0sec 2024-11-24T02:47:49,113 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:49,114 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:B 2024-11-24T02:47:49,114 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:47:49,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:49,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:49,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,124 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:47:49,124 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/C is initiating minor compaction (all files) 2024-11-24T02:47:49,124 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/C in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,124 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/3737dabd772941d2a1b065154ef87091, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/13c1a2990ed44ddfbfc2b320a4a86d09, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4d861d44c6dd496f9220deb2caf24702] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=32.9 K 2024-11-24T02:47:49,124 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3737dabd772941d2a1b065154ef87091, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732416466369 2024-11-24T02:47:49,125 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13c1a2990ed44ddfbfc2b320a4a86d09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732416466441 2024-11-24T02:47:49,125 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d861d44c6dd496f9220deb2caf24702, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732416467664 2024-11-24T02:47:49,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742191_1367 (size=12154) 2024-11-24T02:47:49,138 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:49,141 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b8dc11a2397f45e08a651e9207a1e3b9_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b8dc11a2397f45e08a651e9207a1e3b9_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:49,141 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/6769dd381af44611a8af1d35897828f1, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:49,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/6769dd381af44611a8af1d35897828f1 is 175, key is test_row_0/A:col10/1732416469040/Put/seqid=0 2024-11-24T02:47:49,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742192_1368 (size=31165) 2024-11-24T02:47:49,152 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#C#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:49,153 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/96a15689de094fa4b2f0273c1e656363 is 50, key is test_row_0/C:col10/1732416467664/Put/seqid=0 2024-11-24T02:47:49,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742193_1369 (size=30955) 2024-11-24T02:47:49,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742194_1370 (size=12104) 2024-11-24T02:47:49,169 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/96a15689de094fa4b2f0273c1e656363 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/96a15689de094fa4b2f0273c1e656363 2024-11-24T02:47:49,175 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 859cf8a736ca56969dfab9f223200140/C of 859cf8a736ca56969dfab9f223200140 into 96a15689de094fa4b2f0273c1e656363(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:49,175 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:49,175 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/C, priority=13, startTime=1732416469035; duration=0sec 2024-11-24T02:47:49,175 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:49,175 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:C 2024-11-24T02:47:49,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416529186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416529197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416529197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416529197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:49,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:49,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416529407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416529416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416529416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416529416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:49,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,551 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/036b74ef82504473a23fcb075de9b103 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/036b74ef82504473a23fcb075de9b103 2024-11-24T02:47:49,556 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 859cf8a736ca56969dfab9f223200140/A of 859cf8a736ca56969dfab9f223200140 into 036b74ef82504473a23fcb075de9b103(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:49,556 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:49,556 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/A, priority=13, startTime=1732416469032; duration=0sec 2024-11-24T02:47:49,556 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:49,556 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:A 2024-11-24T02:47:49,561 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/6769dd381af44611a8af1d35897828f1 2024-11-24T02:47:49,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/65ace6c42dc6401dac0dcb8be39a7050 is 50, key is test_row_0/B:col10/1732416469040/Put/seqid=0 2024-11-24T02:47:49,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:49,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742195_1371 (size=12001) 2024-11-24T02:47:49,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416529716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416529727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416529737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416529737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:49,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:49,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:49,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:49,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/65ace6c42dc6401dac0dcb8be39a7050 2024-11-24T02:47:49,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/fc111e6dda76495190a56f6b8de626c5 is 50, key is test_row_0/C:col10/1732416469040/Put/seqid=0 2024-11-24T02:47:49,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742196_1372 (size=12001) 2024-11-24T02:47:50,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:50,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416530087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:50,093 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:50,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:50,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416530237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:50,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,242 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:50,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416530246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416530256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416530256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:50,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:50,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:50,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/fc111e6dda76495190a56f6b8de626c5 2024-11-24T02:47:50,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/6769dd381af44611a8af1d35897828f1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/6769dd381af44611a8af1d35897828f1 2024-11-24T02:47:50,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/6769dd381af44611a8af1d35897828f1, entries=150, sequenceid=79, filesize=30.2 K 2024-11-24T02:47:50,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/65ace6c42dc6401dac0dcb8be39a7050 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/65ace6c42dc6401dac0dcb8be39a7050 2024-11-24T02:47:50,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/65ace6c42dc6401dac0dcb8be39a7050, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T02:47:50,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/fc111e6dda76495190a56f6b8de626c5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fc111e6dda76495190a56f6b8de626c5 2024-11-24T02:47:50,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fc111e6dda76495190a56f6b8de626c5, entries=150, sequenceid=79, filesize=11.7 K 2024-11-24T02:47:50,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 859cf8a736ca56969dfab9f223200140 in 1360ms, sequenceid=79, compaction requested=false 2024-11-24T02:47:50,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:50,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-24T02:47:50,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:50,500 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244f0b8d229c864240a5ea6dedba2471a8_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416469077/Put/seqid=0 2024-11-24T02:47:50,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742197_1373 (size=12154) 2024-11-24T02:47:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:50,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:50,915 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244f0b8d229c864240a5ea6dedba2471a8_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f0b8d229c864240a5ea6dedba2471a8_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:50,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/02f6d65c20374740992a2b0133208675, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/02f6d65c20374740992a2b0133208675 is 175, key is test_row_0/A:col10/1732416469077/Put/seqid=0 2024-11-24T02:47:50,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742198_1374 (size=30955) 2024-11-24T02:47:50,934 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/02f6d65c20374740992a2b0133208675 2024-11-24T02:47:50,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/64356eecaec7421da3c6f0ee8c9e84d9 is 50, key is test_row_0/B:col10/1732416469077/Put/seqid=0 2024-11-24T02:47:50,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742199_1375 (size=12001) 2024-11-24T02:47:50,964 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/64356eecaec7421da3c6f0ee8c9e84d9 2024-11-24T02:47:50,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/ce7ac3e3fec6440988d33009e47bde6f is 50, key is test_row_0/C:col10/1732416469077/Put/seqid=0 2024-11-24T02:47:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742200_1376 (size=12001) 2024-11-24T02:47:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:51,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:51,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416531306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416531307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416531321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416531322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,405 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/ce7ac3e3fec6440988d33009e47bde6f 2024-11-24T02:47:51,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/02f6d65c20374740992a2b0133208675 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/02f6d65c20374740992a2b0133208675 2024-11-24T02:47:51,412 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/02f6d65c20374740992a2b0133208675, entries=150, sequenceid=93, filesize=30.2 K 2024-11-24T02:47:51,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/64356eecaec7421da3c6f0ee8c9e84d9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/64356eecaec7421da3c6f0ee8c9e84d9 2024-11-24T02:47:51,416 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/64356eecaec7421da3c6f0ee8c9e84d9, entries=150, sequenceid=93, filesize=11.7 K 2024-11-24T02:47:51,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/ce7ac3e3fec6440988d33009e47bde6f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/ce7ac3e3fec6440988d33009e47bde6f 2024-11-24T02:47:51,419 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/ce7ac3e3fec6440988d33009e47bde6f, entries=150, sequenceid=93, filesize=11.7 K 2024-11-24T02:47:51,420 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 859cf8a736ca56969dfab9f223200140 in 920ms, sequenceid=93, compaction requested=true 2024-11-24T02:47:51,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:51,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:51,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-24T02:47:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-24T02:47:51,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-24T02:47:51,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.0700 sec 2024-11-24T02:47:51,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 5.0750 sec 2024-11-24T02:47:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:51,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:51,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:47:51,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112404336e5ce486410e8dc5f102ab5d734f_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416471320/Put/seqid=0 2024-11-24T02:47:51,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742201_1377 (size=14594) 2024-11-24T02:47:51,444 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:51,447 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112404336e5ce486410e8dc5f102ab5d734f_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404336e5ce486410e8dc5f102ab5d734f_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:51,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e7b29042046c4b2b918473239ccdd53b, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:51,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e7b29042046c4b2b918473239ccdd53b is 175, key is test_row_0/A:col10/1732416471320/Put/seqid=0 2024-11-24T02:47:51,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742202_1378 (size=39549) 2024-11-24T02:47:51,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416531444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416531445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416531446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416531452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416531556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416531567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416531567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416531567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416531766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416531777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416531777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416531777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:51,851 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e7b29042046c4b2b918473239ccdd53b 2024-11-24T02:47:51,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/a157c4b0855a4a3081203e5a9ac3d368 is 50, key is test_row_0/B:col10/1732416471320/Put/seqid=0 2024-11-24T02:47:51,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742203_1379 (size=12001) 2024-11-24T02:47:51,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/a157c4b0855a4a3081203e5a9ac3d368 2024-11-24T02:47:51,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/80a8f492f772488094dd02ac0caba75b is 50, key is test_row_0/C:col10/1732416471320/Put/seqid=0 2024-11-24T02:47:51,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742204_1380 (size=12001) 2024-11-24T02:47:52,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416532076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416532086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416532086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416532086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416532107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,113 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:52,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/80a8f492f772488094dd02ac0caba75b 2024-11-24T02:47:52,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e7b29042046c4b2b918473239ccdd53b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e7b29042046c4b2b918473239ccdd53b 2024-11-24T02:47:52,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e7b29042046c4b2b918473239ccdd53b, entries=200, sequenceid=120, filesize=38.6 K 2024-11-24T02:47:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/a157c4b0855a4a3081203e5a9ac3d368 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/a157c4b0855a4a3081203e5a9ac3d368 2024-11-24T02:47:52,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/a157c4b0855a4a3081203e5a9ac3d368, entries=150, sequenceid=120, filesize=11.7 K 2024-11-24T02:47:52,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/80a8f492f772488094dd02ac0caba75b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/80a8f492f772488094dd02ac0caba75b 2024-11-24T02:47:52,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/80a8f492f772488094dd02ac0caba75b, entries=150, sequenceid=120, filesize=11.7 K 2024-11-24T02:47:52,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 859cf8a736ca56969dfab9f223200140 in 855ms, sequenceid=120, compaction requested=true 2024-11-24T02:47:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:52,286 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:52,286 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:52,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:52,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:52,287 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:52,287 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132624 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:52,287 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/B is initiating minor compaction (all files) 2024-11-24T02:47:52,287 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/A is initiating minor compaction (all files) 2024-11-24T02:47:52,287 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/B in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:52,287 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/A in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:52,287 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/297e9098f6c64246860e8f55dcb9307d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/65ace6c42dc6401dac0dcb8be39a7050, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/64356eecaec7421da3c6f0ee8c9e84d9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/a157c4b0855a4a3081203e5a9ac3d368] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=47.0 K 2024-11-24T02:47:52,287 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/036b74ef82504473a23fcb075de9b103, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/6769dd381af44611a8af1d35897828f1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/02f6d65c20374740992a2b0133208675, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e7b29042046c4b2b918473239ccdd53b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=129.5 K 2024-11-24T02:47:52,287 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:52,287 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/036b74ef82504473a23fcb075de9b103, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/6769dd381af44611a8af1d35897828f1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/02f6d65c20374740992a2b0133208675, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e7b29042046c4b2b918473239ccdd53b] 2024-11-24T02:47:52,287 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 297e9098f6c64246860e8f55dcb9307d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732416467090 2024-11-24T02:47:52,288 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036b74ef82504473a23fcb075de9b103, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732416467090 2024-11-24T02:47:52,288 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ace6c42dc6401dac0dcb8be39a7050, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416467888 2024-11-24T02:47:52,288 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6769dd381af44611a8af1d35897828f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416467888 2024-11-24T02:47:52,288 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 64356eecaec7421da3c6f0ee8c9e84d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732416469063 2024-11-24T02:47:52,288 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f6d65c20374740992a2b0133208675, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732416469063 2024-11-24T02:47:52,288 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting a157c4b0855a4a3081203e5a9ac3d368, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732416471320 2024-11-24T02:47:52,288 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b29042046c4b2b918473239ccdd53b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732416471305 2024-11-24T02:47:52,295 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:52,300 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#B#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:52,301 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/2719914f55dd43c08d6f1ffab585c151 is 50, key is test_row_0/B:col10/1732416471320/Put/seqid=0 2024-11-24T02:47:52,303 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411247ba7362c7c144bbfb8b139496b4b13cb_859cf8a736ca56969dfab9f223200140 store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:52,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742205_1381 (size=12241) 2024-11-24T02:47:52,305 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411247ba7362c7c144bbfb8b139496b4b13cb_859cf8a736ca56969dfab9f223200140, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:52,305 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411247ba7362c7c144bbfb8b139496b4b13cb_859cf8a736ca56969dfab9f223200140 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742206_1382 (size=4469) 2024-11-24T02:47:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:52,602 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:52,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:47:52,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112446564f506d91435f9aebc62793c94629_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416472600/Put/seqid=0 2024-11-24T02:47:52,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742207_1383 (size=19624) 2024-11-24T02:47:52,620 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:52,623 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112446564f506d91435f9aebc62793c94629_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112446564f506d91435f9aebc62793c94629_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:52,624 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/681586efa5714371861b365a9fe0eb0d, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:52,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/681586efa5714371861b365a9fe0eb0d is 175, key is test_row_0/A:col10/1732416472600/Put/seqid=0 2024-11-24T02:47:52,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742208_1384 (size=56883) 2024-11-24T02:47:52,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416532656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416532661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416532662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416532663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,708 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/2719914f55dd43c08d6f1ffab585c151 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2719914f55dd43c08d6f1ffab585c151 2024-11-24T02:47:52,709 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#A#compaction#323 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:52,709 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e6a6b6eb3bcc4fffb06382d5a810bb8a is 175, key is test_row_0/A:col10/1732416471320/Put/seqid=0 2024-11-24T02:47:52,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742209_1385 (size=31195) 2024-11-24T02:47:52,717 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/B of 859cf8a736ca56969dfab9f223200140 into 2719914f55dd43c08d6f1ffab585c151(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:52,717 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:52,717 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/B, priority=12, startTime=1732416472286; duration=0sec 2024-11-24T02:47:52,717 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:52,717 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:B 2024-11-24T02:47:52,717 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:52,718 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:52,718 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/C is initiating minor compaction (all files) 2024-11-24T02:47:52,718 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/C in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:52,718 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e6a6b6eb3bcc4fffb06382d5a810bb8a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e6a6b6eb3bcc4fffb06382d5a810bb8a 2024-11-24T02:47:52,718 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/96a15689de094fa4b2f0273c1e656363, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fc111e6dda76495190a56f6b8de626c5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/ce7ac3e3fec6440988d33009e47bde6f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/80a8f492f772488094dd02ac0caba75b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=47.0 K 2024-11-24T02:47:52,719 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 96a15689de094fa4b2f0273c1e656363, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732416467090 2024-11-24T02:47:52,719 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting fc111e6dda76495190a56f6b8de626c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732416467888 2024-11-24T02:47:52,719 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting ce7ac3e3fec6440988d33009e47bde6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732416469063 2024-11-24T02:47:52,720 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a8f492f772488094dd02ac0caba75b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732416471320 2024-11-24T02:47:52,722 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/A of 859cf8a736ca56969dfab9f223200140 into e6a6b6eb3bcc4fffb06382d5a810bb8a(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:52,722 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:52,722 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/A, priority=12, startTime=1732416472286; duration=0sec 2024-11-24T02:47:52,722 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:52,722 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:A 2024-11-24T02:47:52,727 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#C#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:52,727 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/498c457c36224cd4a1d77dfa02e8160c is 50, key is test_row_0/C:col10/1732416471320/Put/seqid=0 2024-11-24T02:47:52,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742210_1386 (size=12241) 2024-11-24T02:47:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416532767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416532776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416532776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416532777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416532977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:52,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416532986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416532996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416532996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/681586efa5714371861b365a9fe0eb0d 2024-11-24T02:47:53,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/cc0f2d2f7a524b0e866a69b97e4bb7b7 is 50, key is test_row_0/B:col10/1732416472600/Put/seqid=0 2024-11-24T02:47:53,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742211_1387 (size=12101) 2024-11-24T02:47:53,138 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/498c457c36224cd4a1d77dfa02e8160c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/498c457c36224cd4a1d77dfa02e8160c 2024-11-24T02:47:53,142 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/C of 859cf8a736ca56969dfab9f223200140 into 498c457c36224cd4a1d77dfa02e8160c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:53,142 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:53,142 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/C, priority=12, startTime=1732416472286; duration=0sec 2024-11-24T02:47:53,142 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:53,142 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:C 2024-11-24T02:47:53,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416533287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416533297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416533306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416533306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/cc0f2d2f7a524b0e866a69b97e4bb7b7 2024-11-24T02:47:53,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/9ad6ad4be9c94404a0b7609903674bb1 is 50, key is test_row_0/C:col10/1732416472600/Put/seqid=0 2024-11-24T02:47:53,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742212_1388 (size=12101) 2024-11-24T02:47:53,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416533797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416533816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416533816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416533816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:53,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:53,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/9ad6ad4be9c94404a0b7609903674bb1 2024-11-24T02:47:53,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/681586efa5714371861b365a9fe0eb0d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/681586efa5714371861b365a9fe0eb0d 2024-11-24T02:47:53,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/681586efa5714371861b365a9fe0eb0d, entries=300, sequenceid=132, filesize=55.5 K 2024-11-24T02:47:53,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/cc0f2d2f7a524b0e866a69b97e4bb7b7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/cc0f2d2f7a524b0e866a69b97e4bb7b7 2024-11-24T02:47:53,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/cc0f2d2f7a524b0e866a69b97e4bb7b7, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T02:47:53,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/9ad6ad4be9c94404a0b7609903674bb1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/9ad6ad4be9c94404a0b7609903674bb1 2024-11-24T02:47:53,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/9ad6ad4be9c94404a0b7609903674bb1, entries=150, sequenceid=132, filesize=11.8 K 2024-11-24T02:47:53,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 859cf8a736ca56969dfab9f223200140 in 1272ms, sequenceid=132, compaction requested=false 2024-11-24T02:47:53,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-24T02:47:54,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:54,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-24T02:47:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:47:54,539 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:54,540 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:54,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:47:54,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T02:47:54,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:54,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:47:54,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d26ce4165b98451e9ca758cb6683ec3e_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416472644/Put/seqid=0 2024-11-24T02:47:54,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742213_1389 (size=12304) 2024-11-24T02:47:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:54,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:54,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416534842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:54,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416534842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:54,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416534843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416534844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:47:54,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416534956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416534956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416534957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416534957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:54,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:55,106 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d26ce4165b98451e9ca758cb6683ec3e_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d26ce4165b98451e9ca758cb6683ec3e_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:55,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/766b8d5fde414700848ff7f51d43f757, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:55,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/766b8d5fde414700848ff7f51d43f757 is 175, key is test_row_0/A:col10/1732416472644/Put/seqid=0 2024-11-24T02:47:55,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742214_1390 (size=31105) 2024-11-24T02:47:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:47:55,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416535166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416535167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416535177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416535177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416535477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416535477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416535486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416535486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,512 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/766b8d5fde414700848ff7f51d43f757 2024-11-24T02:47:55,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f9051946bb9f42e6aa107ab8762d22c8 is 50, key is test_row_0/B:col10/1732416472644/Put/seqid=0 2024-11-24T02:47:55,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742215_1391 (size=12151) 2024-11-24T02:47:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:47:55,982 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f9051946bb9f42e6aa107ab8762d22c8 2024-11-24T02:47:55,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/f46a6def984445e491fc6dc17bd0709e is 50, key is test_row_0/C:col10/1732416472644/Put/seqid=0 2024-11-24T02:47:55,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742216_1392 (size=12151) 2024-11-24T02:47:55,992 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/f46a6def984445e491fc6dc17bd0709e 2024-11-24T02:47:55,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416535986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416535986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:55,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:55,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/766b8d5fde414700848ff7f51d43f757 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/766b8d5fde414700848ff7f51d43f757 2024-11-24T02:47:55,998 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/766b8d5fde414700848ff7f51d43f757, entries=150, sequenceid=159, filesize=30.4 K 2024-11-24T02:47:55,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f9051946bb9f42e6aa107ab8762d22c8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f9051946bb9f42e6aa107ab8762d22c8 2024-11-24T02:47:56,002 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f9051946bb9f42e6aa107ab8762d22c8, entries=150, sequenceid=159, filesize=11.9 K 2024-11-24T02:47:56,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/f46a6def984445e491fc6dc17bd0709e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/f46a6def984445e491fc6dc17bd0709e 2024-11-24T02:47:56,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416535997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:56,005 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/f46a6def984445e491fc6dc17bd0709e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-24T02:47:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416535997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:56,006 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 859cf8a736ca56969dfab9f223200140 in 1314ms, sequenceid=159, compaction requested=true 2024-11-24T02:47:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-24T02:47:56,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-24T02:47:56,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-24T02:47:56,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4670 sec 2024-11-24T02:47:56,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4710 sec 2024-11-24T02:47:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:56,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:47:56,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e6de4b8c567a4536944c4425585f5396_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416474843/Put/seqid=0 2024-11-24T02:47:56,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742217_1393 (size=14794) 2024-11-24T02:47:56,164 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:56,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e6de4b8c567a4536944c4425585f5396_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e6de4b8c567a4536944c4425585f5396_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:56,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/b76a3a9de9f9489f8a32c6b95e47daea, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:56,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/b76a3a9de9f9489f8a32c6b95e47daea is 175, key is test_row_0/A:col10/1732416474843/Put/seqid=0 2024-11-24T02:47:56,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742218_1394 (size=39749) 2024-11-24T02:47:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416536300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:56,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416536417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:56,571 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/b76a3a9de9f9489f8a32c6b95e47daea 2024-11-24T02:47:56,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/25efaef61e2844f882accc41ead54cfe is 50, key is test_row_0/B:col10/1732416474843/Put/seqid=0 2024-11-24T02:47:56,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742219_1395 (size=12151) 2024-11-24T02:47:56,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416536636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T02:47:56,686 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:56,687 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-24T02:47:56,688 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:47:56,689 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:56,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:47:56,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T02:47:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:56,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:56,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:56,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:56,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416536946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:56,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:56,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/25efaef61e2844f882accc41ead54cfe 2024-11-24T02:47:56,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/4ee1da9e90464d4c935a4918c311159f is 50, key is test_row_0/C:col10/1732416474843/Put/seqid=0 2024-11-24T02:47:56,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742220_1396 (size=12151) 2024-11-24T02:47:56,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T02:47:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:56,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:56,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:57,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:47:57,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416537007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:57,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416537007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:57,017 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:57,017 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:57,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416537016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:57,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416537016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:57,024 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:57,024 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:47:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:57,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T02:47:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:57,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:57,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T02:47:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:47:57,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/4ee1da9e90464d4c935a4918c311159f 2024-11-24T02:47:57,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/b76a3a9de9f9489f8a32c6b95e47daea as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/b76a3a9de9f9489f8a32c6b95e47daea 2024-11-24T02:47:57,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/b76a3a9de9f9489f8a32c6b95e47daea, entries=200, sequenceid=172, filesize=38.8 K 2024-11-24T02:47:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/25efaef61e2844f882accc41ead54cfe as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/25efaef61e2844f882accc41ead54cfe 2024-11-24T02:47:57,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/25efaef61e2844f882accc41ead54cfe, entries=150, sequenceid=172, filesize=11.9 K 2024-11-24T02:47:57,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/4ee1da9e90464d4c935a4918c311159f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4ee1da9e90464d4c935a4918c311159f 2024-11-24T02:47:57,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4ee1da9e90464d4c935a4918c311159f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-24T02:47:57,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 859cf8a736ca56969dfab9f223200140 in 1262ms, sequenceid=172, compaction requested=true 2024-11-24T02:47:57,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:57,416 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:57,418 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158932 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:57,418 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/A is initiating minor compaction (all files) 2024-11-24T02:47:57,418 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/A in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,418 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e6a6b6eb3bcc4fffb06382d5a810bb8a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/681586efa5714371861b365a9fe0eb0d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/766b8d5fde414700848ff7f51d43f757, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/b76a3a9de9f9489f8a32c6b95e47daea] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=155.2 K 2024-11-24T02:47:57,418 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,418 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e6a6b6eb3bcc4fffb06382d5a810bb8a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/681586efa5714371861b365a9fe0eb0d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/766b8d5fde414700848ff7f51d43f757, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/b76a3a9de9f9489f8a32c6b95e47daea] 2024-11-24T02:47:57,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:57,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:57,419 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:57,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:57,422 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6a6b6eb3bcc4fffb06382d5a810bb8a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732416471320 2024-11-24T02:47:57,422 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681586efa5714371861b365a9fe0eb0d, keycount=300, bloomtype=ROW, size=55.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732416471444 2024-11-24T02:47:57,422 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:57,423 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/B is initiating minor compaction (all files) 2024-11-24T02:47:57,423 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/B in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,423 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2719914f55dd43c08d6f1ffab585c151, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/cc0f2d2f7a524b0e866a69b97e4bb7b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f9051946bb9f42e6aa107ab8762d22c8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/25efaef61e2844f882accc41ead54cfe] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=47.5 K 2024-11-24T02:47:57,423 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 766b8d5fde414700848ff7f51d43f757, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732416472644 2024-11-24T02:47:57,424 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2719914f55dd43c08d6f1ffab585c151, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732416471320 2024-11-24T02:47:57,424 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b76a3a9de9f9489f8a32c6b95e47daea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732416474841 2024-11-24T02:47:57,425 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0f2d2f7a524b0e866a69b97e4bb7b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732416471450 2024-11-24T02:47:57,425 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f9051946bb9f42e6aa107ab8762d22c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732416472644 2024-11-24T02:47:57,426 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 25efaef61e2844f882accc41ead54cfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732416474842 2024-11-24T02:47:57,437 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:57,438 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#B#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:57,439 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/04b45adb01e54809b9058a8210d9c45d is 50, key is test_row_0/B:col10/1732416474843/Put/seqid=0 2024-11-24T02:47:57,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T02:47:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,453 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T02:47:57,460 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124d1b9f7dfdf6f4121accaa90989f1f428_859cf8a736ca56969dfab9f223200140 store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:57,462 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124d1b9f7dfdf6f4121accaa90989f1f428_859cf8a736ca56969dfab9f223200140, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:57,462 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d1b9f7dfdf6f4121accaa90989f1f428_859cf8a736ca56969dfab9f223200140 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:57,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742221_1397 (size=12527) 2024-11-24T02:47:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:57,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:57,467 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/04b45adb01e54809b9058a8210d9c45d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/04b45adb01e54809b9058a8210d9c45d 2024-11-24T02:47:57,471 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/B of 859cf8a736ca56969dfab9f223200140 into 04b45adb01e54809b9058a8210d9c45d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:57,471 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:57,471 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/B, priority=12, startTime=1732416477418; duration=0sec 2024-11-24T02:47:57,471 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:57,471 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:B 2024-11-24T02:47:57,471 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:57,472 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:57,472 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/C is initiating minor compaction (all files) 2024-11-24T02:47:57,472 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/C in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:57,472 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/498c457c36224cd4a1d77dfa02e8160c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/9ad6ad4be9c94404a0b7609903674bb1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/f46a6def984445e491fc6dc17bd0709e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4ee1da9e90464d4c935a4918c311159f] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=47.5 K 2024-11-24T02:47:57,473 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 498c457c36224cd4a1d77dfa02e8160c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732416471320 2024-11-24T02:47:57,473 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad6ad4be9c94404a0b7609903674bb1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732416471450 2024-11-24T02:47:57,473 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f46a6def984445e491fc6dc17bd0709e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732416472644 2024-11-24T02:47:57,473 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee1da9e90464d4c935a4918c311159f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732416474842 2024-11-24T02:47:57,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bbe1541628e84a65b15f3b9fc9600c30_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416476290/Put/seqid=0 2024-11-24T02:47:57,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742222_1398 (size=4469) 2024-11-24T02:47:57,477 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#A#compaction#336 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:57,478 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/43a6b7ad4ec64bce8c8819d5f5014be1 is 175, key is test_row_0/A:col10/1732416474843/Put/seqid=0 2024-11-24T02:47:57,488 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:57,488 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/b570326c4c7e45ccbae1851f7fb58837 is 50, key is test_row_0/C:col10/1732416474843/Put/seqid=0 2024-11-24T02:47:57,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742223_1399 (size=12304) 2024-11-24T02:47:57,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742224_1400 (size=31481) 2024-11-24T02:47:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742225_1401 (size=12527) 2024-11-24T02:47:57,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416537535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:57,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416537657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:57,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:47:57,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416537867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:57,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:57,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:57,890 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bbe1541628e84a65b15f3b9fc9600c30_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bbe1541628e84a65b15f3b9fc9600c30_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/63f15cd98ea341cd87fc67b4e91fe2ab, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:57,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/63f15cd98ea341cd87fc67b4e91fe2ab is 175, key is test_row_0/A:col10/1732416476290/Put/seqid=0 2024-11-24T02:47:57,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742226_1402 (size=31105) 2024-11-24T02:47:57,900 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/43a6b7ad4ec64bce8c8819d5f5014be1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43a6b7ad4ec64bce8c8819d5f5014be1 2024-11-24T02:47:57,903 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/A of 859cf8a736ca56969dfab9f223200140 into 43a6b7ad4ec64bce8c8819d5f5014be1(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:57,903 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:57,903 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/A, priority=12, startTime=1732416477416; duration=0sec 2024-11-24T02:47:57,903 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:57,903 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:A 2024-11-24T02:47:57,905 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/b570326c4c7e45ccbae1851f7fb58837 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/b570326c4c7e45ccbae1851f7fb58837 2024-11-24T02:47:57,909 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/C of 859cf8a736ca56969dfab9f223200140 into b570326c4c7e45ccbae1851f7fb58837(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:47:57,909 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:57,909 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/C, priority=12, startTime=1732416477419; duration=0sec 2024-11-24T02:47:57,909 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:57,909 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:C 2024-11-24T02:47:58,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416538177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:58,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:58,295 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/63f15cd98ea341cd87fc67b4e91fe2ab 2024-11-24T02:47:58,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/4d5d677b89bf47388c4ae04a279a76f5 is 50, key is test_row_0/B:col10/1732416476290/Put/seqid=0 2024-11-24T02:47:58,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742227_1403 (size=12151) 2024-11-24T02:47:58,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416538697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:58,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:58,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/4d5d677b89bf47388c4ae04a279a76f5 2024-11-24T02:47:58,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/8ddef9ea08174c4c91cc6e52dfbfe033 is 50, key is test_row_0/C:col10/1732416476290/Put/seqid=0 2024-11-24T02:47:58,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742228_1404 (size=12151) 2024-11-24T02:47:58,715 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/8ddef9ea08174c4c91cc6e52dfbfe033 2024-11-24T02:47:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/63f15cd98ea341cd87fc67b4e91fe2ab as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/63f15cd98ea341cd87fc67b4e91fe2ab 2024-11-24T02:47:58,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/63f15cd98ea341cd87fc67b4e91fe2ab, entries=150, sequenceid=195, filesize=30.4 K 2024-11-24T02:47:58,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/4d5d677b89bf47388c4ae04a279a76f5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/4d5d677b89bf47388c4ae04a279a76f5 2024-11-24T02:47:58,728 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/4d5d677b89bf47388c4ae04a279a76f5, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T02:47:58,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/8ddef9ea08174c4c91cc6e52dfbfe033 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ddef9ea08174c4c91cc6e52dfbfe033 2024-11-24T02:47:58,731 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ddef9ea08174c4c91cc6e52dfbfe033, entries=150, sequenceid=195, filesize=11.9 K 2024-11-24T02:47:58,732 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 859cf8a736ca56969dfab9f223200140 in 1279ms, sequenceid=195, compaction requested=false 2024-11-24T02:47:58,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:58,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:58,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-24T02:47:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-24T02:47:58,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-24T02:47:58,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0440 sec 2024-11-24T02:47:58,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0480 sec 2024-11-24T02:47:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T02:47:58,837 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:58,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-24T02:47:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:47:58,839 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:58,840 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:58,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:47:58,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T02:47:58,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:58,993 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:47:59,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112497f1415751074edc959e331e73ef388d_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416477484/Put/seqid=0 2024-11-24T02:47:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:59,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:47:59,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742229_1405 (size=12304) 2024-11-24T02:47:59,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416539108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416539111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416539112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416539112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:47:59,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416539236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416539237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416539237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416539237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416539457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:47:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416539457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416539457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416539457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:59,475 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112497f1415751074edc959e331e73ef388d_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112497f1415751074edc959e331e73ef388d_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e281907897714cafa42ec5a0c3f4a000, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:59,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e281907897714cafa42ec5a0c3f4a000 is 175, key is test_row_0/A:col10/1732416477484/Put/seqid=0 2024-11-24T02:47:59,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742230_1406 (size=31105) 2024-11-24T02:47:59,490 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e281907897714cafa42ec5a0c3f4a000 2024-11-24T02:47:59,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/8cdbc408f5f9464790096eed884385f0 is 50, key is test_row_0/B:col10/1732416477484/Put/seqid=0 2024-11-24T02:47:59,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742231_1407 (size=12151) 2024-11-24T02:47:59,510 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/8cdbc408f5f9464790096eed884385f0 2024-11-24T02:47:59,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/d08f9fd71d874377a3842f6eb8e7eb7c is 50, key is test_row_0/C:col10/1732416477484/Put/seqid=0 2024-11-24T02:47:59,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742232_1408 (size=12151) 2024-11-24T02:47:59,547 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/d08f9fd71d874377a3842f6eb8e7eb7c 2024-11-24T02:47:59,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e281907897714cafa42ec5a0c3f4a000 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e281907897714cafa42ec5a0c3f4a000 2024-11-24T02:47:59,563 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e281907897714cafa42ec5a0c3f4a000, entries=150, sequenceid=211, filesize=30.4 K 2024-11-24T02:47:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/8cdbc408f5f9464790096eed884385f0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8cdbc408f5f9464790096eed884385f0 2024-11-24T02:47:59,573 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8cdbc408f5f9464790096eed884385f0, entries=150, sequenceid=211, filesize=11.9 K 2024-11-24T02:47:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/d08f9fd71d874377a3842f6eb8e7eb7c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d08f9fd71d874377a3842f6eb8e7eb7c 2024-11-24T02:47:59,579 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d08f9fd71d874377a3842f6eb8e7eb7c, entries=150, sequenceid=211, filesize=11.9 K 2024-11-24T02:47:59,580 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 859cf8a736ca56969dfab9f223200140 in 587ms, sequenceid=211, compaction requested=true 2024-11-24T02:47:59,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:59,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:59,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-24T02:47:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-24T02:47:59,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-24T02:47:59,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 742 msec 2024-11-24T02:47:59,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 748 msec 2024-11-24T02:47:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:59,730 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:59,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:47:59,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bad8f1c4d1f440a2a72ef1238c06f898_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416479727/Put/seqid=0 2024-11-24T02:47:59,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742233_1409 (size=14794) 2024-11-24T02:47:59,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:59,763 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bad8f1c4d1f440a2a72ef1238c06f898_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bad8f1c4d1f440a2a72ef1238c06f898_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:59,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/43c9fc01be1140b2a2748e320b2f0ea9, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:59,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/43c9fc01be1140b2a2748e320b2f0ea9 is 175, key is test_row_0/A:col10/1732416479727/Put/seqid=0 2024-11-24T02:47:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742234_1410 (size=39749) 2024-11-24T02:47:59,795 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/43c9fc01be1140b2a2748e320b2f0ea9 2024-11-24T02:47:59,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416539789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416539790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416539792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/9f4dac3d3c334df0843248e8b0ed18d0 is 50, key is test_row_0/B:col10/1732416479727/Put/seqid=0 2024-11-24T02:47:59,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742235_1411 (size=12151) 2024-11-24T02:47:59,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416539810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:47:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416539812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:47:59,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/9f4dac3d3c334df0843248e8b0ed18d0 2024-11-24T02:47:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:47:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:47:59,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/e6f07b67c2234b2ea597564a53c30edc is 50, key is test_row_0/C:col10/1732416479727/Put/seqid=0 2024-11-24T02:47:59,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742236_1412 (size=12151) 2024-11-24T02:47:59,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/e6f07b67c2234b2ea597564a53c30edc 2024-11-24T02:47:59,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/43c9fc01be1140b2a2748e320b2f0ea9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43c9fc01be1140b2a2748e320b2f0ea9 2024-11-24T02:47:59,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43c9fc01be1140b2a2748e320b2f0ea9, entries=200, sequenceid=235, filesize=38.8 K 2024-11-24T02:47:59,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/9f4dac3d3c334df0843248e8b0ed18d0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/9f4dac3d3c334df0843248e8b0ed18d0 2024-11-24T02:47:59,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/9f4dac3d3c334df0843248e8b0ed18d0, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T02:47:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/e6f07b67c2234b2ea597564a53c30edc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e6f07b67c2234b2ea597564a53c30edc 2024-11-24T02:47:59,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e6f07b67c2234b2ea597564a53c30edc, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T02:47:59,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 859cf8a736ca56969dfab9f223200140 in 189ms, sequenceid=235, compaction requested=true 2024-11-24T02:47:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:47:59,919 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:47:59,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:59,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:47:59,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:47:59,920 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:47:59,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:47:59,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:47:59,920 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:59,920 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/A is initiating minor compaction (all files) 2024-11-24T02:47:59,921 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/A in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:59,921 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43a6b7ad4ec64bce8c8819d5f5014be1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/63f15cd98ea341cd87fc67b4e91fe2ab, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e281907897714cafa42ec5a0c3f4a000, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43c9fc01be1140b2a2748e320b2f0ea9] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=130.3 K 2024-11-24T02:47:59,921 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:59,921 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43a6b7ad4ec64bce8c8819d5f5014be1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/63f15cd98ea341cd87fc67b4e91fe2ab, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e281907897714cafa42ec5a0c3f4a000, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43c9fc01be1140b2a2748e320b2f0ea9] 2024-11-24T02:47:59,921 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:47:59,921 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/B is initiating minor compaction (all files) 2024-11-24T02:47:59,921 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/B in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:47:59,921 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a6b7ad4ec64bce8c8819d5f5014be1, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732416474842 2024-11-24T02:47:59,921 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/04b45adb01e54809b9058a8210d9c45d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/4d5d677b89bf47388c4ae04a279a76f5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8cdbc408f5f9464790096eed884385f0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/9f4dac3d3c334df0843248e8b0ed18d0] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=47.8 K 2024-11-24T02:47:59,921 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f15cd98ea341cd87fc67b4e91fe2ab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732416476268 2024-11-24T02:47:59,921 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b45adb01e54809b9058a8210d9c45d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732416474842 2024-11-24T02:47:59,922 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d5d677b89bf47388c4ae04a279a76f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732416476268 2024-11-24T02:47:59,922 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e281907897714cafa42ec5a0c3f4a000, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416477484 2024-11-24T02:47:59,922 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cdbc408f5f9464790096eed884385f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416477484 2024-11-24T02:47:59,922 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c9fc01be1140b2a2748e320b2f0ea9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416479106 2024-11-24T02:47:59,922 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f4dac3d3c334df0843248e8b0ed18d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416479110 2024-11-24T02:47:59,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:47:59,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:47:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:59,934 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#B#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:47:59,935 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/6c105eaf2abf4ab2b48a087f35d95b57 is 50, key is test_row_0/B:col10/1732416479727/Put/seqid=0 2024-11-24T02:47:59,941 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:59,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245c0d946ede064b9686a42228f728cf37_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416479930/Put/seqid=0 2024-11-24T02:47:59,974 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112460930a68ff4c4ee1b3ae5f9252e17ff8_859cf8a736ca56969dfab9f223200140 store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:59,976 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112460930a68ff4c4ee1b3ae5f9252e17ff8_859cf8a736ca56969dfab9f223200140, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T02:47:59,977 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:47:59,977 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112460930a68ff4c4ee1b3ae5f9252e17ff8_859cf8a736ca56969dfab9f223200140 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:59,980 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:47:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-24T02:47:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:47:59,982 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:47:59,984 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:47:59,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:47:59,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742238_1414 (size=14794) 2024-11-24T02:47:59,989 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:47:59,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742237_1413 (size=12663) 2024-11-24T02:47:59,993 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245c0d946ede064b9686a42228f728cf37_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245c0d946ede064b9686a42228f728cf37_859cf8a736ca56969dfab9f223200140 2024-11-24T02:47:59,994 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0d4a9b9098c8457fb8d70979960d165c, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:47:59,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0d4a9b9098c8457fb8d70979960d165c is 175, key is test_row_0/A:col10/1732416479930/Put/seqid=0 2024-11-24T02:48:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416539998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416539999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416540001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416540003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416540004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742240_1416 (size=39749) 2024-11-24T02:48:00,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0d4a9b9098c8457fb8d70979960d165c 2024-11-24T02:48:00,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/5febe6264da94f59893a40df84030bb6 is 50, key is test_row_0/B:col10/1732416479930/Put/seqid=0 2024-11-24T02:48:00,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742239_1415 (size=4469) 2024-11-24T02:48:00,057 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#A#compaction#348 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:00,058 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e608e257607c41ef8f9da80d7fbd86b6 is 175, key is test_row_0/A:col10/1732416479727/Put/seqid=0 2024-11-24T02:48:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:48:00,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742241_1417 (size=12151) 2024-11-24T02:48:00,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742242_1418 (size=31617) 2024-11-24T02:48:00,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/5febe6264da94f59893a40df84030bb6 2024-11-24T02:48:00,097 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/e608e257607c41ef8f9da80d7fbd86b6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e608e257607c41ef8f9da80d7fbd86b6 2024-11-24T02:48:00,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/fd7a391381f441b7bd28d432f0c292e9 is 50, key is test_row_0/C:col10/1732416479930/Put/seqid=0 2024-11-24T02:48:00,106 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/A of 859cf8a736ca56969dfab9f223200140 into e608e257607c41ef8f9da80d7fbd86b6(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:00,106 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:00,106 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/A, priority=12, startTime=1732416479919; duration=0sec 2024-11-24T02:48:00,107 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:00,107 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:A 2024-11-24T02:48:00,107 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:00,108 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:00,108 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/C is initiating minor compaction (all files) 2024-11-24T02:48:00,108 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/C in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:00,108 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/b570326c4c7e45ccbae1851f7fb58837, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ddef9ea08174c4c91cc6e52dfbfe033, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d08f9fd71d874377a3842f6eb8e7eb7c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e6f07b67c2234b2ea597564a53c30edc] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=47.8 K 2024-11-24T02:48:00,109 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b570326c4c7e45ccbae1851f7fb58837, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732416474842 2024-11-24T02:48:00,109 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ddef9ea08174c4c91cc6e52dfbfe033, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732416476268 2024-11-24T02:48:00,111 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d08f9fd71d874377a3842f6eb8e7eb7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732416477484 2024-11-24T02:48:00,111 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f07b67c2234b2ea597564a53c30edc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416479110 2024-11-24T02:48:00,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416540127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416540127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416540127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416540128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T02:48:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:00,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416540128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:00,139 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#C#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:00,139 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/cf1f7832b16a4d64af5a013a98ad7bb3 is 50, key is test_row_0/C:col10/1732416479727/Put/seqid=0 2024-11-24T02:48:00,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742243_1419 (size=12151) 2024-11-24T02:48:00,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/fd7a391381f441b7bd28d432f0c292e9 2024-11-24T02:48:00,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0d4a9b9098c8457fb8d70979960d165c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0d4a9b9098c8457fb8d70979960d165c 2024-11-24T02:48:00,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742244_1420 (size=12663) 2024-11-24T02:48:00,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0d4a9b9098c8457fb8d70979960d165c, entries=200, sequenceid=249, filesize=38.8 K 2024-11-24T02:48:00,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/5febe6264da94f59893a40df84030bb6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/5febe6264da94f59893a40df84030bb6 2024-11-24T02:48:00,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/5febe6264da94f59893a40df84030bb6, entries=150, sequenceid=249, filesize=11.9 K 2024-11-24T02:48:00,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/fd7a391381f441b7bd28d432f0c292e9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fd7a391381f441b7bd28d432f0c292e9 2024-11-24T02:48:00,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fd7a391381f441b7bd28d432f0c292e9, entries=150, sequenceid=249, filesize=11.9 K 2024-11-24T02:48:00,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 859cf8a736ca56969dfab9f223200140 in 284ms, sequenceid=249, compaction requested=false 2024-11-24T02:48:00,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:00,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T02:48:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:00,291 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:48:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:48:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112403763bbe6e0d4bb5ad3ccf8e07f5f399_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416480002/Put/seqid=0 2024-11-24T02:48:00,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742245_1421 (size=12454) 2024-11-24T02:48:00,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:00,328 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112403763bbe6e0d4bb5ad3ccf8e07f5f399_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112403763bbe6e0d4bb5ad3ccf8e07f5f399_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/8240d173c47444838221562164d2e11d, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:00,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/8240d173c47444838221562164d2e11d is 175, key is test_row_0/A:col10/1732416480002/Put/seqid=0 2024-11-24T02:48:00,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:00,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742246_1422 (size=31255) 2024-11-24T02:48:00,352 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/8240d173c47444838221562164d2e11d 2024-11-24T02:48:00,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/b73e62c048764c2bba1c01520606ee74 is 50, key is test_row_0/B:col10/1732416480002/Put/seqid=0 2024-11-24T02:48:00,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416540375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416540375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742247_1423 (size=12301) 2024-11-24T02:48:00,396 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/6c105eaf2abf4ab2b48a087f35d95b57 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/6c105eaf2abf4ab2b48a087f35d95b57 2024-11-24T02:48:00,400 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/B of 859cf8a736ca56969dfab9f223200140 into 6c105eaf2abf4ab2b48a087f35d95b57(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:00,400 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:00,400 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/B, priority=12, startTime=1732416479919; duration=0sec 2024-11-24T02:48:00,400 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:00,400 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:B 2024-11-24T02:48:00,404 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/b73e62c048764c2bba1c01520606ee74 2024-11-24T02:48:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416540389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416540390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416540390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/d5dd96ea39a54d5ba51d9eacedcfb095 is 50, key is test_row_0/C:col10/1732416480002/Put/seqid=0 2024-11-24T02:48:00,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742248_1424 (size=12301) 2024-11-24T02:48:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416540497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416540497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416540518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416540518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416540518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,601 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/cf1f7832b16a4d64af5a013a98ad7bb3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/cf1f7832b16a4d64af5a013a98ad7bb3 2024-11-24T02:48:00,605 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/C of 859cf8a736ca56969dfab9f223200140 into cf1f7832b16a4d64af5a013a98ad7bb3(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:00,605 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:00,605 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/C, priority=12, startTime=1732416479920; duration=0sec 2024-11-24T02:48:00,605 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:00,606 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:C 2024-11-24T02:48:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:48:00,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416540706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416540716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416540749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416540749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416540749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:00,815 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/d5dd96ea39a54d5ba51d9eacedcfb095 2024-11-24T02:48:00,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/8240d173c47444838221562164d2e11d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/8240d173c47444838221562164d2e11d 2024-11-24T02:48:00,823 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/8240d173c47444838221562164d2e11d, entries=150, sequenceid=272, filesize=30.5 K 2024-11-24T02:48:00,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/b73e62c048764c2bba1c01520606ee74 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/b73e62c048764c2bba1c01520606ee74 2024-11-24T02:48:00,828 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/b73e62c048764c2bba1c01520606ee74, entries=150, sequenceid=272, filesize=12.0 K 2024-11-24T02:48:00,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/d5dd96ea39a54d5ba51d9eacedcfb095 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d5dd96ea39a54d5ba51d9eacedcfb095 2024-11-24T02:48:00,831 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d5dd96ea39a54d5ba51d9eacedcfb095, entries=150, sequenceid=272, filesize=12.0 K 2024-11-24T02:48:00,832 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 859cf8a736ca56969dfab9f223200140 in 541ms, sequenceid=272, compaction requested=true 2024-11-24T02:48:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:00,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-24T02:48:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-24T02:48:00,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-24T02:48:00,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 849 msec 2024-11-24T02:48:00,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 855 msec 2024-11-24T02:48:01,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:01,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:01,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T02:48:01,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242bd65f1cabe24800a24cdf74748918e9_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416481030/Put/seqid=0 2024-11-24T02:48:01,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742249_1425 (size=17534) 2024-11-24T02:48:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416541079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416541081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416541087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416541088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416541095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T02:48:01,117 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:01,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-24T02:48:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T02:48:01,119 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:01,119 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:01,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:01,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416541206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416541206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416541206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416541206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416541217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T02:48:01,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:01,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416541417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416541417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416541417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416541417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:01,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T02:48:01,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416541426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,496 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:01,501 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242bd65f1cabe24800a24cdf74748918e9_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242bd65f1cabe24800a24cdf74748918e9_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:01,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/eb60b0f31b8d436c8b787f85ef7273cc, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:01,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/eb60b0f31b8d436c8b787f85ef7273cc is 175, key is test_row_0/A:col10/1732416481030/Put/seqid=0 2024-11-24T02:48:01,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742250_1426 (size=48639) 2024-11-24T02:48:01,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:01,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:01,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416541726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416541727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416541727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416541727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T02:48:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416541746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:01,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:01,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:01,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:01,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:01,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/eb60b0f31b8d436c8b787f85ef7273cc 2024-11-24T02:48:01,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/fa141090d81a4f04b78c350eb128e0f1 is 50, key is test_row_0/B:col10/1732416481030/Put/seqid=0 2024-11-24T02:48:01,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742251_1427 (size=12301) 2024-11-24T02:48:02,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:02,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:02,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:02,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416542237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:02,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416542246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416542246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416542247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T02:48:02,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416542257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:02,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/fa141090d81a4f04b78c350eb128e0f1 2024-11-24T02:48:02,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/e48de2a7f9dd46eea094086ac0352281 is 50, key is test_row_0/C:col10/1732416481030/Put/seqid=0 2024-11-24T02:48:02,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742252_1428 (size=12301) 2024-11-24T02:48:02,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:02,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:02,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/e48de2a7f9dd46eea094086ac0352281 2024-11-24T02:48:02,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/eb60b0f31b8d436c8b787f85ef7273cc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eb60b0f31b8d436c8b787f85ef7273cc 2024-11-24T02:48:02,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eb60b0f31b8d436c8b787f85ef7273cc, entries=250, sequenceid=290, filesize=47.5 K 2024-11-24T02:48:02,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/fa141090d81a4f04b78c350eb128e0f1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/fa141090d81a4f04b78c350eb128e0f1 2024-11-24T02:48:02,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/fa141090d81a4f04b78c350eb128e0f1, entries=150, sequenceid=290, filesize=12.0 K 2024-11-24T02:48:02,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/e48de2a7f9dd46eea094086ac0352281 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e48de2a7f9dd46eea094086ac0352281 2024-11-24T02:48:02,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e48de2a7f9dd46eea094086ac0352281, entries=150, sequenceid=290, filesize=12.0 K 2024-11-24T02:48:02,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 859cf8a736ca56969dfab9f223200140 in 1717ms, sequenceid=290, compaction requested=true 2024-11-24T02:48:02,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:02,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:02,748 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:02,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:02,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:02,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:02,748 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:02,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:02,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:02,749 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:02,749 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:02,749 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/A is initiating minor compaction (all files) 2024-11-24T02:48:02,749 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/B is initiating minor compaction (all files) 2024-11-24T02:48:02,749 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/A in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,749 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/B in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,750 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e608e257607c41ef8f9da80d7fbd86b6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0d4a9b9098c8457fb8d70979960d165c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/8240d173c47444838221562164d2e11d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eb60b0f31b8d436c8b787f85ef7273cc] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=147.7 K 2024-11-24T02:48:02,750 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/6c105eaf2abf4ab2b48a087f35d95b57, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/5febe6264da94f59893a40df84030bb6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/b73e62c048764c2bba1c01520606ee74, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/fa141090d81a4f04b78c350eb128e0f1] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=48.3 K 2024-11-24T02:48:02,750 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e608e257607c41ef8f9da80d7fbd86b6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0d4a9b9098c8457fb8d70979960d165c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/8240d173c47444838221562164d2e11d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eb60b0f31b8d436c8b787f85ef7273cc] 2024-11-24T02:48:02,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e608e257607c41ef8f9da80d7fbd86b6, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416479110 2024-11-24T02:48:02,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c105eaf2abf4ab2b48a087f35d95b57, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416479110 2024-11-24T02:48:02,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d4a9b9098c8457fb8d70979960d165c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732416479788 2024-11-24T02:48:02,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5febe6264da94f59893a40df84030bb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732416479788 2024-11-24T02:48:02,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8240d173c47444838221562164d2e11d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732416479995 2024-11-24T02:48:02,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b73e62c048764c2bba1c01520606ee74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732416479995 2024-11-24T02:48:02,751 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb60b0f31b8d436c8b787f85ef7273cc, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732416480362 2024-11-24T02:48:02,751 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting fa141090d81a4f04b78c350eb128e0f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732416480388 2024-11-24T02:48:02,756 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:02,758 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#B#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:02,758 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112454485d40162a4f15b5ae157c2bd1309e_859cf8a736ca56969dfab9f223200140 store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:02,758 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/2d7ed90220c7412b8bd67e04a918b8d5 is 50, key is test_row_0/B:col10/1732416481030/Put/seqid=0 2024-11-24T02:48:02,761 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112454485d40162a4f15b5ae157c2bd1309e_859cf8a736ca56969dfab9f223200140, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:02,761 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112454485d40162a4f15b5ae157c2bd1309e_859cf8a736ca56969dfab9f223200140 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:02,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742253_1429 (size=12949) 2024-11-24T02:48:02,775 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/2d7ed90220c7412b8bd67e04a918b8d5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2d7ed90220c7412b8bd67e04a918b8d5 2024-11-24T02:48:02,779 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/B of 859cf8a736ca56969dfab9f223200140 into 2d7ed90220c7412b8bd67e04a918b8d5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:02,779 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:02,779 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/B, priority=12, startTime=1732416482748; duration=0sec 2024-11-24T02:48:02,779 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:02,779 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:B 2024-11-24T02:48:02,779 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:02,780 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:02,780 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/C is initiating minor compaction (all files) 2024-11-24T02:48:02,780 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/C in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,781 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/cf1f7832b16a4d64af5a013a98ad7bb3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fd7a391381f441b7bd28d432f0c292e9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d5dd96ea39a54d5ba51d9eacedcfb095, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e48de2a7f9dd46eea094086ac0352281] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=48.3 K 2024-11-24T02:48:02,781 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting cf1f7832b16a4d64af5a013a98ad7bb3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732416479110 2024-11-24T02:48:02,781 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting fd7a391381f441b7bd28d432f0c292e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732416479788 2024-11-24T02:48:02,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742254_1430 (size=4469) 2024-11-24T02:48:02,782 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d5dd96ea39a54d5ba51d9eacedcfb095, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732416479995 2024-11-24T02:48:02,782 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e48de2a7f9dd46eea094086ac0352281, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732416480388 2024-11-24T02:48:02,782 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#A#compaction#360 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:02,783 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/eed0b16e8db84a35a5d03f7159382c36 is 175, key is test_row_0/A:col10/1732416481030/Put/seqid=0 2024-11-24T02:48:02,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742255_1431 (size=31903) 2024-11-24T02:48:02,789 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#C#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:02,790 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/0f1b9c9136fb4e5daee3270f7d04347e is 50, key is test_row_0/C:col10/1732416481030/Put/seqid=0 2024-11-24T02:48:02,792 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/eed0b16e8db84a35a5d03f7159382c36 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eed0b16e8db84a35a5d03f7159382c36 2024-11-24T02:48:02,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742256_1432 (size=12949) 2024-11-24T02:48:02,797 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/A of 859cf8a736ca56969dfab9f223200140 into eed0b16e8db84a35a5d03f7159382c36(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:02,797 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:02,797 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/A, priority=12, startTime=1732416482748; duration=0sec 2024-11-24T02:48:02,797 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:02,797 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:A 2024-11-24T02:48:02,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T02:48:02,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:02,800 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T02:48:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246c188a0ec5bf4bc0bd4fd71eeacabd12_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416481093/Put/seqid=0 2024-11-24T02:48:02,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742257_1433 (size=12454) 2024-11-24T02:48:02,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:02,822 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246c188a0ec5bf4bc0bd4fd71eeacabd12_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c188a0ec5bf4bc0bd4fd71eeacabd12_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:02,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/014b72a9145f4e749f4f00ae4f82b04f, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:02,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/014b72a9145f4e749f4f00ae4f82b04f is 175, key is test_row_0/A:col10/1732416481093/Put/seqid=0 2024-11-24T02:48:02,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742258_1434 (size=31255) 2024-11-24T02:48:03,199 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/0f1b9c9136fb4e5daee3270f7d04347e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/0f1b9c9136fb4e5daee3270f7d04347e 2024-11-24T02:48:03,203 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 859cf8a736ca56969dfab9f223200140/C of 859cf8a736ca56969dfab9f223200140 into 0f1b9c9136fb4e5daee3270f7d04347e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:03,203 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:03,203 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/C, priority=12, startTime=1732416482748; duration=0sec 2024-11-24T02:48:03,203 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:03,203 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:C 2024-11-24T02:48:03,230 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/014b72a9145f4e749f4f00ae4f82b04f 2024-11-24T02:48:03,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/556e3daf21b742bfa5be1023ca0ad8e3 is 50, key is test_row_0/B:col10/1732416481093/Put/seqid=0 2024-11-24T02:48:03,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742259_1435 (size=12301) 2024-11-24T02:48:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:03,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T02:48:03,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416543276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416543281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,286 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416543282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416543283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416543285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416543396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416543396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416543396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416543396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416543606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416543606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416543606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416543606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,639 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/556e3daf21b742bfa5be1023ca0ad8e3 2024-11-24T02:48:03,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/01fcf7e790f245a48a130f56f67a8004 is 50, key is test_row_0/C:col10/1732416481093/Put/seqid=0 2024-11-24T02:48:03,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742260_1436 (size=12301) 2024-11-24T02:48:03,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416543917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416543917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416543917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416543917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:03,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,052 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/01fcf7e790f245a48a130f56f67a8004 2024-11-24T02:48:04,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/014b72a9145f4e749f4f00ae4f82b04f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/014b72a9145f4e749f4f00ae4f82b04f 2024-11-24T02:48:04,058 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/014b72a9145f4e749f4f00ae4f82b04f, entries=150, sequenceid=312, filesize=30.5 K 2024-11-24T02:48:04,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/556e3daf21b742bfa5be1023ca0ad8e3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/556e3daf21b742bfa5be1023ca0ad8e3 2024-11-24T02:48:04,062 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/556e3daf21b742bfa5be1023ca0ad8e3, entries=150, sequenceid=312, filesize=12.0 K 2024-11-24T02:48:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/01fcf7e790f245a48a130f56f67a8004 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/01fcf7e790f245a48a130f56f67a8004 2024-11-24T02:48:04,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/01fcf7e790f245a48a130f56f67a8004, entries=150, sequenceid=312, filesize=12.0 K 2024-11-24T02:48:04,067 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 859cf8a736ca56969dfab9f223200140 in 1267ms, sequenceid=312, compaction requested=false 2024-11-24T02:48:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-24T02:48:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-24T02:48:04,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-24T02:48:04,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9490 sec 2024-11-24T02:48:04,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.9530 sec 2024-11-24T02:48:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:04,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:04,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-24T02:48:04,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124259a26bc0c3f4a0597cd6ab2a1966136_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416483284/Put/seqid=0 2024-11-24T02:48:04,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742261_1437 (size=17534) 2024-11-24T02:48:04,446 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:04,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124259a26bc0c3f4a0597cd6ab2a1966136_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124259a26bc0c3f4a0597cd6ab2a1966136_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:04,450 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/206520d569fa428eaa5711fea50621e3, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:04,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/206520d569fa428eaa5711fea50621e3 is 175, key is test_row_0/A:col10/1732416483284/Put/seqid=0 2024-11-24T02:48:04,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742262_1438 (size=48639) 2024-11-24T02:48:04,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416544477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416544478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416544479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416544481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416544586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416544596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416544596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416544597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416544796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416544806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416544806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416544817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:04,859 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/206520d569fa428eaa5711fea50621e3 2024-11-24T02:48:04,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/8eace66965e241368ce7d950421f5394 is 50, key is test_row_0/B:col10/1732416483284/Put/seqid=0 2024-11-24T02:48:04,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742263_1439 (size=12301) 2024-11-24T02:48:04,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/8eace66965e241368ce7d950421f5394 2024-11-24T02:48:04,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/8ccc1d564caa4b52b421d394de6ccffe is 50, key is test_row_0/C:col10/1732416483284/Put/seqid=0 2024-11-24T02:48:04,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742264_1440 (size=12301) 2024-11-24T02:48:05,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416545107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416545116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416545116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416545136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T02:48:05,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:05,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-24T02:48:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:48:05,279 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:05,279 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:05,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:05,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/8ccc1d564caa4b52b421d394de6ccffe 2024-11-24T02:48:05,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/206520d569fa428eaa5711fea50621e3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/206520d569fa428eaa5711fea50621e3 2024-11-24T02:48:05,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/206520d569fa428eaa5711fea50621e3, entries=250, sequenceid=331, filesize=47.5 K 2024-11-24T02:48:05,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/8eace66965e241368ce7d950421f5394 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8eace66965e241368ce7d950421f5394 2024-11-24T02:48:05,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8eace66965e241368ce7d950421f5394, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:48:05,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/8ccc1d564caa4b52b421d394de6ccffe as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ccc1d564caa4b52b421d394de6ccffe 2024-11-24T02:48:05,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ccc1d564caa4b52b421d394de6ccffe, entries=150, sequenceid=331, filesize=12.0 K 2024-11-24T02:48:05,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 859cf8a736ca56969dfab9f223200140 in 868ms, sequenceid=331, compaction requested=true 2024-11-24T02:48:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:05,299 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:05,299 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 859cf8a736ca56969dfab9f223200140:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:05,300 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:05,300 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/A is initiating minor compaction (all files) 2024-11-24T02:48:05,300 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/A in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,300 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eed0b16e8db84a35a5d03f7159382c36, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/014b72a9145f4e749f4f00ae4f82b04f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/206520d569fa428eaa5711fea50621e3] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=109.2 K 2024-11-24T02:48:05,300 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,300 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eed0b16e8db84a35a5d03f7159382c36, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/014b72a9145f4e749f4f00ae4f82b04f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/206520d569fa428eaa5711fea50621e3] 2024-11-24T02:48:05,301 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:05,301 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/B is initiating minor compaction (all files) 2024-11-24T02:48:05,301 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed0b16e8db84a35a5d03f7159382c36, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732416480388 2024-11-24T02:48:05,301 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/B in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,301 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2d7ed90220c7412b8bd67e04a918b8d5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/556e3daf21b742bfa5be1023ca0ad8e3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8eace66965e241368ce7d950421f5394] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=36.7 K 2024-11-24T02:48:05,301 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 014b72a9145f4e749f4f00ae4f82b04f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732416481080 2024-11-24T02:48:05,301 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 206520d569fa428eaa5711fea50621e3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416483273 2024-11-24T02:48:05,302 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d7ed90220c7412b8bd67e04a918b8d5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732416480388 2024-11-24T02:48:05,302 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 556e3daf21b742bfa5be1023ca0ad8e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732416481080 2024-11-24T02:48:05,302 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eace66965e241368ce7d950421f5394, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416483284 2024-11-24T02:48:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:05,306 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:05,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-24T02:48:05,307 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:05,309 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#B#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:05,309 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/35b7b81246ce44e88697ecb6a71cec06 is 50, key is test_row_0/B:col10/1732416483284/Put/seqid=0 2024-11-24T02:48:05,309 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411241f467e776b944df4ac73f0cbaacb4562_859cf8a736ca56969dfab9f223200140 store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:05,311 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411241f467e776b944df4ac73f0cbaacb4562_859cf8a736ca56969dfab9f223200140, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:05,311 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241f467e776b944df4ac73f0cbaacb4562_859cf8a736ca56969dfab9f223200140 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:05,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742266_1442 (size=4469) 2024-11-24T02:48:05,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112456d564375d0245f2808118690fa87b27_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416485304/Put/seqid=0 2024-11-24T02:48:05,322 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#A#compaction#369 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:05,323 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/5d178c9212b24df5aae08c627831357c is 175, key is test_row_0/A:col10/1732416483284/Put/seqid=0 2024-11-24T02:48:05,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742265_1441 (size=13051) 2024-11-24T02:48:05,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742268_1444 (size=32005) 2024-11-24T02:48:05,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742267_1443 (size=14994) 2024-11-24T02:48:05,328 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/35b7b81246ce44e88697ecb6a71cec06 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/35b7b81246ce44e88697ecb6a71cec06 2024-11-24T02:48:05,333 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/5d178c9212b24df5aae08c627831357c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/5d178c9212b24df5aae08c627831357c 2024-11-24T02:48:05,336 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 859cf8a736ca56969dfab9f223200140/B of 859cf8a736ca56969dfab9f223200140 into 35b7b81246ce44e88697ecb6a71cec06(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:05,336 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:05,336 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/B, priority=13, startTime=1732416485299; duration=0sec 2024-11-24T02:48:05,336 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:05,336 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:B 2024-11-24T02:48:05,336 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:05,337 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 859cf8a736ca56969dfab9f223200140/A of 859cf8a736ca56969dfab9f223200140 into 5d178c9212b24df5aae08c627831357c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:05,337 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:05,337 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/A, priority=13, startTime=1732416485299; duration=0sec 2024-11-24T02:48:05,337 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:05,337 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:A 2024-11-24T02:48:05,337 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:05,337 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 859cf8a736ca56969dfab9f223200140/C is initiating minor compaction (all files) 2024-11-24T02:48:05,337 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 859cf8a736ca56969dfab9f223200140/C in TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,338 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/0f1b9c9136fb4e5daee3270f7d04347e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/01fcf7e790f245a48a130f56f67a8004, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ccc1d564caa4b52b421d394de6ccffe] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp, totalSize=36.7 K 2024-11-24T02:48:05,338 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f1b9c9136fb4e5daee3270f7d04347e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732416480388 2024-11-24T02:48:05,338 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 01fcf7e790f245a48a130f56f67a8004, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732416481080 2024-11-24T02:48:05,340 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ccc1d564caa4b52b421d394de6ccffe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732416483284 2024-11-24T02:48:05,345 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 859cf8a736ca56969dfab9f223200140#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:05,345 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/7b6db45c617b4ea78e3c80783fbd3986 is 50, key is test_row_0/C:col10/1732416483284/Put/seqid=0 2024-11-24T02:48:05,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742269_1445 (size=13051) 2024-11-24T02:48:05,355 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/7b6db45c617b4ea78e3c80783fbd3986 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/7b6db45c617b4ea78e3c80783fbd3986 2024-11-24T02:48:05,360 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 859cf8a736ca56969dfab9f223200140/C of 859cf8a736ca56969dfab9f223200140 into 7b6db45c617b4ea78e3c80783fbd3986(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:05,360 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:05,360 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., storeName=859cf8a736ca56969dfab9f223200140/C, priority=13, startTime=1732416485299; duration=0sec 2024-11-24T02:48:05,361 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:05,361 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 859cf8a736ca56969dfab9f223200140:C 2024-11-24T02:48:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:48:05,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416545405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416545517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:48:05,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416545617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416545636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416545636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416545657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,727 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:05,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112456d564375d0245f2808118690fa87b27_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456d564375d0245f2808118690fa87b27_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:05,733 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0534b8cc899e4406aebb42b7ac44ef75, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:05,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0534b8cc899e4406aebb42b7ac44ef75 is 175, key is test_row_0/A:col10/1732416485304/Put/seqid=0 2024-11-24T02:48:05,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:05,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416545738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:05,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:05,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742270_1446 (size=39949) 2024-11-24T02:48:05,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:05,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:05,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:48:06,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:06,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:06,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416546046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:06,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0534b8cc899e4406aebb42b7ac44ef75 2024-11-24T02:48:06,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/d179246638f24fbcae9e0c473b2c89a4 is 50, key is test_row_0/B:col10/1732416485304/Put/seqid=0 2024-11-24T02:48:06,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742271_1447 (size=12301) 2024-11-24T02:48:06,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:06,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:06,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:06,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:06,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,407 INFO [Thread-1614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1614. 2024-11-24T02:48:06,407 DEBUG [Thread-1614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:06,407 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-11-24T02:48:06,407 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,407 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:06,407 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:06,408 DEBUG [Thread-1614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,408 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,408 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:06,410 INFO [Thread-1616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1616. 2024-11-24T02:48:06,410 DEBUG [Thread-1616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:06,410 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,410 DEBUG [Thread-1616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:06,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:48:06,416 INFO [Thread-1612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1612. 2024-11-24T02:48:06,417 DEBUG [Thread-1612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:06,417 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,417 DEBUG [Thread-1612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:06,418 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-11-24T02:48:06,418 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:06,418 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,418 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:06,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:06,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:06,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:06,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42636 deadline: 1732416546556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:06,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/d179246638f24fbcae9e0c473b2c89a4 2024-11-24T02:48:06,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/7d9268bbacc94868af9a68155b6f9f7f is 50, key is test_row_0/C:col10/1732416485304/Put/seqid=0 2024-11-24T02:48:06,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742272_1448 (size=12301) 2024-11-24T02:48:06,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42670 deadline: 1732416546637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:06,638 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42696 deadline: 1732416546657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42658 deadline: 1732416546657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:06,659 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:06,659 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:06,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1732416546677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:06,678 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 859cf8a736ca56969dfab9f223200140 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:06,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:06,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:06,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:06,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:06,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/7d9268bbacc94868af9a68155b6f9f7f 2024-11-24T02:48:06,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/0534b8cc899e4406aebb42b7ac44ef75 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0534b8cc899e4406aebb42b7ac44ef75 2024-11-24T02:48:06,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0534b8cc899e4406aebb42b7ac44ef75, entries=200, sequenceid=351, filesize=39.0 K 2024-11-24T02:48:06,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/d179246638f24fbcae9e0c473b2c89a4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/d179246638f24fbcae9e0c473b2c89a4 2024-11-24T02:48:07,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/d179246638f24fbcae9e0c473b2c89a4, entries=150, sequenceid=351, filesize=12.0 K 2024-11-24T02:48:07,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/7d9268bbacc94868af9a68155b6f9f7f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/7d9268bbacc94868af9a68155b6f9f7f 2024-11-24T02:48:07,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/7d9268bbacc94868af9a68155b6f9f7f, entries=150, sequenceid=351, filesize=12.0 K 2024-11-24T02:48:07,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 859cf8a736ca56969dfab9f223200140 in 1701ms, sequenceid=351, compaction requested=false 2024-11-24T02:48:07,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:07,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T02:48:07,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:07,009 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-24T02:48:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241d7ba76a31ab45a3a67f5fe423695bbe_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416485403/Put/seqid=0 2024-11-24T02:48:07,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742273_1449 (size=12454) 2024-11-24T02:48:07,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:07,427 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241d7ba76a31ab45a3a67f5fe423695bbe_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241d7ba76a31ab45a3a67f5fe423695bbe_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:48:07,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/a50409515de345f88584fe311829191f, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/a50409515de345f88584fe311829191f is 175, key is test_row_0/A:col10/1732416485403/Put/seqid=0 2024-11-24T02:48:07,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742274_1450 (size=31255) 2024-11-24T02:48:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:07,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. as already flushing 2024-11-24T02:48:07,572 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-24T02:48:07,572 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:07,572 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:07,573 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:07,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:07,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/a50409515de345f88584fe311829191f 2024-11-24T02:48:07,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f8557863d536401d9d2eae97a1d869e0 is 50, key is test_row_0/B:col10/1732416485403/Put/seqid=0 2024-11-24T02:48:07,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742275_1451 (size=12301) 2024-11-24T02:48:08,106 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:48:08,253 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f8557863d536401d9d2eae97a1d869e0 2024-11-24T02:48:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/03ed2f0174cd499ea230e816afb26744 is 50, key is test_row_0/C:col10/1732416485403/Put/seqid=0 2024-11-24T02:48:08,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742276_1452 (size=12301) 2024-11-24T02:48:08,650 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-24T02:48:08,650 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:08,650 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,650 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:08,668 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/03ed2f0174cd499ea230e816afb26744 2024-11-24T02:48:08,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/a50409515de345f88584fe311829191f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/a50409515de345f88584fe311829191f 2024-11-24T02:48:08,679 INFO [Thread-1603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1603. 2024-11-24T02:48:08,679 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-11-24T02:48:08,679 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:08,679 DEBUG [Thread-1603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:08,679 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,679 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,679 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,679 DEBUG [Thread-1603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:08,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:08,683 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/a50409515de345f88584fe311829191f, entries=150, sequenceid=370, filesize=30.5 K 2024-11-24T02:48:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/f8557863d536401d9d2eae97a1d869e0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f8557863d536401d9d2eae97a1d869e0 2024-11-24T02:48:08,687 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f8557863d536401d9d2eae97a1d869e0, entries=150, sequenceid=370, filesize=12.0 K 2024-11-24T02:48:08,687 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-11-24T02:48:08,687 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:08,687 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,687 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:08,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:08,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/03ed2f0174cd499ea230e816afb26744 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/03ed2f0174cd499ea230e816afb26744 2024-11-24T02:48:08,692 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/03ed2f0174cd499ea230e816afb26744, entries=150, sequenceid=370, filesize=12.0 K 2024-11-24T02:48:08,693 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=33.54 KB/34350 for 859cf8a736ca56969dfab9f223200140 in 1683ms, sequenceid=370, compaction requested=true 2024-11-24T02:48:08,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 859cf8a736ca56969dfab9f223200140: 2024-11-24T02:48:08,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:08,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-24T02:48:08,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-24T02:48:08,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-24T02:48:08,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4150 sec 2024-11-24T02:48:08,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 3.4190 sec 2024-11-24T02:48:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T02:48:09,447 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1288 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3864 rows 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1299 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3880 rows 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1300 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3900 rows 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1303 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3876 rows 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1318 2024-11-24T02:48:09,447 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3913 rows 2024-11-24T02:48:09,447 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:48:09,447 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:48:09,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:09,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:09,447 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:09,448 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:48:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T02:48:09,452 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416489451"}]},"ts":"1732416489451"} 2024-11-24T02:48:09,453 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:48:09,453 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:48:09,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:48:09,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, UNASSIGN}] 2024-11-24T02:48:09,457 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, UNASSIGN 2024-11-24T02:48:09,458 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:09,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, UNASSIGN because future has completed 2024-11-24T02:48:09,459 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:48:09,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:48:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T02:48:09,613 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:09,613 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:48:09,614 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 859cf8a736ca56969dfab9f223200140, disabling compactions & flushes 2024-11-24T02:48:09,614 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:09,614 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:09,614 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. after waiting 0 ms 2024-11-24T02:48:09,614 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:09,614 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 859cf8a736ca56969dfab9f223200140 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T02:48:09,622 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246a5833ed7ab04e888c97e32a8f58135a_859cf8a736ca56969dfab9f223200140 is 50, key is test_row_0/A:col10/1732416488678/Put/seqid=0 2024-11-24T02:48:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742277_1453 (size=12454) 2024-11-24T02:48:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T02:48:10,027 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:10,036 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246a5833ed7ab04e888c97e32a8f58135a_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a5833ed7ab04e888c97e32a8f58135a_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:10,037 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/c17c354c96694e4a8931a756582526a1, store: [table=TestAcidGuarantees family=A region=859cf8a736ca56969dfab9f223200140] 2024-11-24T02:48:10,038 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/c17c354c96694e4a8931a756582526a1 is 175, key is test_row_0/A:col10/1732416488678/Put/seqid=0 2024-11-24T02:48:10,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742278_1454 (size=31255) 2024-11-24T02:48:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T02:48:10,443 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/c17c354c96694e4a8931a756582526a1 2024-11-24T02:48:10,450 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/53f375de804648f5941a43e5893da7ed is 50, key is test_row_0/B:col10/1732416488678/Put/seqid=0 2024-11-24T02:48:10,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742279_1455 (size=12301) 2024-11-24T02:48:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T02:48:10,858 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/53f375de804648f5941a43e5893da7ed 2024-11-24T02:48:10,864 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/44b6fc1ba4934d3d9226816040813f8f is 50, key is test_row_0/C:col10/1732416488678/Put/seqid=0 2024-11-24T02:48:10,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742280_1456 (size=12301) 2024-11-24T02:48:11,268 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/44b6fc1ba4934d3d9226816040813f8f 2024-11-24T02:48:11,272 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/A/c17c354c96694e4a8931a756582526a1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/c17c354c96694e4a8931a756582526a1 2024-11-24T02:48:11,276 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/c17c354c96694e4a8931a756582526a1, entries=150, sequenceid=378, filesize=30.5 K 2024-11-24T02:48:11,276 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/B/53f375de804648f5941a43e5893da7ed as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/53f375de804648f5941a43e5893da7ed 2024-11-24T02:48:11,280 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/53f375de804648f5941a43e5893da7ed, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T02:48:11,281 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/.tmp/C/44b6fc1ba4934d3d9226816040813f8f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/44b6fc1ba4934d3d9226816040813f8f 2024-11-24T02:48:11,285 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/44b6fc1ba4934d3d9226816040813f8f, entries=150, sequenceid=378, filesize=12.0 K 2024-11-24T02:48:11,286 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 859cf8a736ca56969dfab9f223200140 in 1671ms, sequenceid=378, compaction requested=true 2024-11-24T02:48:11,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/51d977a6bf614a0e923d6031d57b61ff, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/4ea3ffdc3d8f415fa4c28d7194d869b0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/036b74ef82504473a23fcb075de9b103, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/799ebc74ec254b0b83522adc0967f820, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/6769dd381af44611a8af1d35897828f1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/02f6d65c20374740992a2b0133208675, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e7b29042046c4b2b918473239ccdd53b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e6a6b6eb3bcc4fffb06382d5a810bb8a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/681586efa5714371861b365a9fe0eb0d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/766b8d5fde414700848ff7f51d43f757, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/b76a3a9de9f9489f8a32c6b95e47daea, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43a6b7ad4ec64bce8c8819d5f5014be1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/63f15cd98ea341cd87fc67b4e91fe2ab, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e281907897714cafa42ec5a0c3f4a000, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43c9fc01be1140b2a2748e320b2f0ea9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e608e257607c41ef8f9da80d7fbd86b6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0d4a9b9098c8457fb8d70979960d165c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/8240d173c47444838221562164d2e11d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eb60b0f31b8d436c8b787f85ef7273cc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eed0b16e8db84a35a5d03f7159382c36, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/014b72a9145f4e749f4f00ae4f82b04f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/206520d569fa428eaa5711fea50621e3] to archive 2024-11-24T02:48:11,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:48:11,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/51d977a6bf614a0e923d6031d57b61ff to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/51d977a6bf614a0e923d6031d57b61ff 2024-11-24T02:48:11,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/4ea3ffdc3d8f415fa4c28d7194d869b0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/4ea3ffdc3d8f415fa4c28d7194d869b0 2024-11-24T02:48:11,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/036b74ef82504473a23fcb075de9b103 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/036b74ef82504473a23fcb075de9b103 2024-11-24T02:48:11,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/799ebc74ec254b0b83522adc0967f820 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/799ebc74ec254b0b83522adc0967f820 2024-11-24T02:48:11,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/6769dd381af44611a8af1d35897828f1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/6769dd381af44611a8af1d35897828f1 2024-11-24T02:48:11,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/02f6d65c20374740992a2b0133208675 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/02f6d65c20374740992a2b0133208675 2024-11-24T02:48:11,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e7b29042046c4b2b918473239ccdd53b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e7b29042046c4b2b918473239ccdd53b 2024-11-24T02:48:11,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e6a6b6eb3bcc4fffb06382d5a810bb8a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e6a6b6eb3bcc4fffb06382d5a810bb8a 2024-11-24T02:48:11,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/681586efa5714371861b365a9fe0eb0d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/681586efa5714371861b365a9fe0eb0d 2024-11-24T02:48:11,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/766b8d5fde414700848ff7f51d43f757 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/766b8d5fde414700848ff7f51d43f757 2024-11-24T02:48:11,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/b76a3a9de9f9489f8a32c6b95e47daea to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/b76a3a9de9f9489f8a32c6b95e47daea 2024-11-24T02:48:11,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43a6b7ad4ec64bce8c8819d5f5014be1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43a6b7ad4ec64bce8c8819d5f5014be1 2024-11-24T02:48:11,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/63f15cd98ea341cd87fc67b4e91fe2ab to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/63f15cd98ea341cd87fc67b4e91fe2ab 2024-11-24T02:48:11,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e281907897714cafa42ec5a0c3f4a000 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e281907897714cafa42ec5a0c3f4a000 2024-11-24T02:48:11,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43c9fc01be1140b2a2748e320b2f0ea9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/43c9fc01be1140b2a2748e320b2f0ea9 2024-11-24T02:48:11,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e608e257607c41ef8f9da80d7fbd86b6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/e608e257607c41ef8f9da80d7fbd86b6 2024-11-24T02:48:11,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0d4a9b9098c8457fb8d70979960d165c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0d4a9b9098c8457fb8d70979960d165c 2024-11-24T02:48:11,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/8240d173c47444838221562164d2e11d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/8240d173c47444838221562164d2e11d 2024-11-24T02:48:11,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eb60b0f31b8d436c8b787f85ef7273cc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eb60b0f31b8d436c8b787f85ef7273cc 2024-11-24T02:48:11,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eed0b16e8db84a35a5d03f7159382c36 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/eed0b16e8db84a35a5d03f7159382c36 2024-11-24T02:48:11,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/014b72a9145f4e749f4f00ae4f82b04f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/014b72a9145f4e749f4f00ae4f82b04f 2024-11-24T02:48:11,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/206520d569fa428eaa5711fea50621e3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/206520d569fa428eaa5711fea50621e3 2024-11-24T02:48:11,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f5c9467be7f643eb90cc2222f6aef4eb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2e546009c70246d1913ddbf1176f1225, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/297e9098f6c64246860e8f55dcb9307d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/75948c660fc64a6097e72cd8ca8eb019, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/65ace6c42dc6401dac0dcb8be39a7050, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/64356eecaec7421da3c6f0ee8c9e84d9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2719914f55dd43c08d6f1ffab585c151, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/a157c4b0855a4a3081203e5a9ac3d368, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/cc0f2d2f7a524b0e866a69b97e4bb7b7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f9051946bb9f42e6aa107ab8762d22c8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/04b45adb01e54809b9058a8210d9c45d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/25efaef61e2844f882accc41ead54cfe, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/4d5d677b89bf47388c4ae04a279a76f5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8cdbc408f5f9464790096eed884385f0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/6c105eaf2abf4ab2b48a087f35d95b57, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/9f4dac3d3c334df0843248e8b0ed18d0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/5febe6264da94f59893a40df84030bb6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/b73e62c048764c2bba1c01520606ee74, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2d7ed90220c7412b8bd67e04a918b8d5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/fa141090d81a4f04b78c350eb128e0f1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/556e3daf21b742bfa5be1023ca0ad8e3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8eace66965e241368ce7d950421f5394] to archive 2024-11-24T02:48:11,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:48:11,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f5c9467be7f643eb90cc2222f6aef4eb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f5c9467be7f643eb90cc2222f6aef4eb 2024-11-24T02:48:11,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2e546009c70246d1913ddbf1176f1225 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2e546009c70246d1913ddbf1176f1225 2024-11-24T02:48:11,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/297e9098f6c64246860e8f55dcb9307d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/297e9098f6c64246860e8f55dcb9307d 2024-11-24T02:48:11,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/75948c660fc64a6097e72cd8ca8eb019 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/75948c660fc64a6097e72cd8ca8eb019 2024-11-24T02:48:11,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/65ace6c42dc6401dac0dcb8be39a7050 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/65ace6c42dc6401dac0dcb8be39a7050 2024-11-24T02:48:11,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/64356eecaec7421da3c6f0ee8c9e84d9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/64356eecaec7421da3c6f0ee8c9e84d9 2024-11-24T02:48:11,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2719914f55dd43c08d6f1ffab585c151 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2719914f55dd43c08d6f1ffab585c151 2024-11-24T02:48:11,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/a157c4b0855a4a3081203e5a9ac3d368 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/a157c4b0855a4a3081203e5a9ac3d368 2024-11-24T02:48:11,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/cc0f2d2f7a524b0e866a69b97e4bb7b7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/cc0f2d2f7a524b0e866a69b97e4bb7b7 2024-11-24T02:48:11,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f9051946bb9f42e6aa107ab8762d22c8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f9051946bb9f42e6aa107ab8762d22c8 2024-11-24T02:48:11,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/04b45adb01e54809b9058a8210d9c45d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/04b45adb01e54809b9058a8210d9c45d 2024-11-24T02:48:11,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/25efaef61e2844f882accc41ead54cfe to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/25efaef61e2844f882accc41ead54cfe 2024-11-24T02:48:11,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/4d5d677b89bf47388c4ae04a279a76f5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/4d5d677b89bf47388c4ae04a279a76f5 2024-11-24T02:48:11,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8cdbc408f5f9464790096eed884385f0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8cdbc408f5f9464790096eed884385f0 2024-11-24T02:48:11,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/6c105eaf2abf4ab2b48a087f35d95b57 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/6c105eaf2abf4ab2b48a087f35d95b57 2024-11-24T02:48:11,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/9f4dac3d3c334df0843248e8b0ed18d0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/9f4dac3d3c334df0843248e8b0ed18d0 2024-11-24T02:48:11,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/5febe6264da94f59893a40df84030bb6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/5febe6264da94f59893a40df84030bb6 2024-11-24T02:48:11,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/b73e62c048764c2bba1c01520606ee74 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/b73e62c048764c2bba1c01520606ee74 2024-11-24T02:48:11,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2d7ed90220c7412b8bd67e04a918b8d5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/2d7ed90220c7412b8bd67e04a918b8d5 2024-11-24T02:48:11,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/fa141090d81a4f04b78c350eb128e0f1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/fa141090d81a4f04b78c350eb128e0f1 2024-11-24T02:48:11,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/556e3daf21b742bfa5be1023ca0ad8e3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/556e3daf21b742bfa5be1023ca0ad8e3 2024-11-24T02:48:11,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8eace66965e241368ce7d950421f5394 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/8eace66965e241368ce7d950421f5394 2024-11-24T02:48:11,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/3737dabd772941d2a1b065154ef87091, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/13c1a2990ed44ddfbfc2b320a4a86d09, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/96a15689de094fa4b2f0273c1e656363, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4d861d44c6dd496f9220deb2caf24702, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fc111e6dda76495190a56f6b8de626c5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/ce7ac3e3fec6440988d33009e47bde6f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/498c457c36224cd4a1d77dfa02e8160c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/80a8f492f772488094dd02ac0caba75b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/9ad6ad4be9c94404a0b7609903674bb1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/f46a6def984445e491fc6dc17bd0709e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/b570326c4c7e45ccbae1851f7fb58837, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4ee1da9e90464d4c935a4918c311159f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ddef9ea08174c4c91cc6e52dfbfe033, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d08f9fd71d874377a3842f6eb8e7eb7c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/cf1f7832b16a4d64af5a013a98ad7bb3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e6f07b67c2234b2ea597564a53c30edc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fd7a391381f441b7bd28d432f0c292e9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d5dd96ea39a54d5ba51d9eacedcfb095, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/0f1b9c9136fb4e5daee3270f7d04347e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e48de2a7f9dd46eea094086ac0352281, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/01fcf7e790f245a48a130f56f67a8004, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ccc1d564caa4b52b421d394de6ccffe] to archive 2024-11-24T02:48:11,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:48:11,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/3737dabd772941d2a1b065154ef87091 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/3737dabd772941d2a1b065154ef87091 2024-11-24T02:48:11,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/13c1a2990ed44ddfbfc2b320a4a86d09 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/13c1a2990ed44ddfbfc2b320a4a86d09 2024-11-24T02:48:11,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/96a15689de094fa4b2f0273c1e656363 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/96a15689de094fa4b2f0273c1e656363 2024-11-24T02:48:11,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4d861d44c6dd496f9220deb2caf24702 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4d861d44c6dd496f9220deb2caf24702 2024-11-24T02:48:11,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fc111e6dda76495190a56f6b8de626c5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fc111e6dda76495190a56f6b8de626c5 2024-11-24T02:48:11,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/ce7ac3e3fec6440988d33009e47bde6f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/ce7ac3e3fec6440988d33009e47bde6f 2024-11-24T02:48:11,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/498c457c36224cd4a1d77dfa02e8160c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/498c457c36224cd4a1d77dfa02e8160c 2024-11-24T02:48:11,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/80a8f492f772488094dd02ac0caba75b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/80a8f492f772488094dd02ac0caba75b 2024-11-24T02:48:11,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/9ad6ad4be9c94404a0b7609903674bb1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/9ad6ad4be9c94404a0b7609903674bb1 2024-11-24T02:48:11,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/f46a6def984445e491fc6dc17bd0709e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/f46a6def984445e491fc6dc17bd0709e 2024-11-24T02:48:11,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/b570326c4c7e45ccbae1851f7fb58837 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/b570326c4c7e45ccbae1851f7fb58837 2024-11-24T02:48:11,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4ee1da9e90464d4c935a4918c311159f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/4ee1da9e90464d4c935a4918c311159f 2024-11-24T02:48:11,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ddef9ea08174c4c91cc6e52dfbfe033 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ddef9ea08174c4c91cc6e52dfbfe033 2024-11-24T02:48:11,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d08f9fd71d874377a3842f6eb8e7eb7c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d08f9fd71d874377a3842f6eb8e7eb7c 2024-11-24T02:48:11,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/cf1f7832b16a4d64af5a013a98ad7bb3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/cf1f7832b16a4d64af5a013a98ad7bb3 2024-11-24T02:48:11,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e6f07b67c2234b2ea597564a53c30edc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e6f07b67c2234b2ea597564a53c30edc 2024-11-24T02:48:11,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fd7a391381f441b7bd28d432f0c292e9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/fd7a391381f441b7bd28d432f0c292e9 2024-11-24T02:48:11,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d5dd96ea39a54d5ba51d9eacedcfb095 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/d5dd96ea39a54d5ba51d9eacedcfb095 2024-11-24T02:48:11,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/0f1b9c9136fb4e5daee3270f7d04347e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/0f1b9c9136fb4e5daee3270f7d04347e 2024-11-24T02:48:11,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e48de2a7f9dd46eea094086ac0352281 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/e48de2a7f9dd46eea094086ac0352281 2024-11-24T02:48:11,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/01fcf7e790f245a48a130f56f67a8004 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/01fcf7e790f245a48a130f56f67a8004 2024-11-24T02:48:11,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ccc1d564caa4b52b421d394de6ccffe to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/8ccc1d564caa4b52b421d394de6ccffe 2024-11-24T02:48:11,345 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/recovered.edits/381.seqid, newMaxSeqId=381, maxSeqId=4 2024-11-24T02:48:11,345 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. 2024-11-24T02:48:11,345 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 859cf8a736ca56969dfab9f223200140: Waiting for close lock at 1732416489614Running coprocessor pre-close hooks at 1732416489614Disabling compacts and flushes for region at 1732416489614Disabling writes for close at 1732416489614Obtaining lock to block concurrent updates at 1732416489614Preparing flush snapshotting stores in 859cf8a736ca56969dfab9f223200140 at 1732416489614Finished memstore snapshotting TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732416489615 (+1 ms)Flushing stores of TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140. at 1732416489616 (+1 ms)Flushing 859cf8a736ca56969dfab9f223200140/A: creating writer at 1732416489616Flushing mob file 859cf8a736ca56969dfab9f223200140/A: appending metadata at 1732416489622 (+6 ms)Flushing mob file 859cf8a736ca56969dfab9f223200140/A: closing flushed file at 1732416489622Flushing 859cf8a736ca56969dfab9f223200140/A: appending metadata at 1732416490037 (+415 ms)Flushing 859cf8a736ca56969dfab9f223200140/A: closing flushed file at 1732416490037Flushing 859cf8a736ca56969dfab9f223200140/B: creating writer at 1732416490447 (+410 ms)Flushing 859cf8a736ca56969dfab9f223200140/B: appending metadata at 1732416490450 (+3 ms)Flushing 859cf8a736ca56969dfab9f223200140/B: closing flushed file at 1732416490450Flushing 859cf8a736ca56969dfab9f223200140/C: creating writer at 1732416490861 (+411 ms)Flushing 859cf8a736ca56969dfab9f223200140/C: appending metadata at 1732416490864 (+3 ms)Flushing 859cf8a736ca56969dfab9f223200140/C: closing flushed file at 1732416490864Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@603eb54c: reopening flushed file at 1732416491272 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37bb545c: reopening flushed file at 1732416491276 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38e1fe2f: reopening flushed file at 1732416491280 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 859cf8a736ca56969dfab9f223200140 in 1671ms, sequenceid=378, compaction requested=true at 1732416491286 (+6 ms)Writing region close event to WAL at 1732416491343 (+57 ms)Running coprocessor post-close hooks at 1732416491345 (+2 ms)Closed at 1732416491345 2024-11-24T02:48:11,347 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,348 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=859cf8a736ca56969dfab9f223200140, regionState=CLOSED 2024-11-24T02:48:11,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:48:11,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-24T02:48:11,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 859cf8a736ca56969dfab9f223200140, server=7c69a60bd8f6,39461,1732416369827 in 1.8900 sec 2024-11-24T02:48:11,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-24T02:48:11,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=859cf8a736ca56969dfab9f223200140, UNASSIGN in 1.8950 sec 2024-11-24T02:48:11,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-24T02:48:11,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8980 sec 2024-11-24T02:48:11,354 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416491354"}]},"ts":"1732416491354"} 2024-11-24T02:48:11,355 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:48:11,355 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:48:11,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-24T02:48:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T02:48:11,598 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:11,599 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:48:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:11,604 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:48:11,606 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:11,609 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,613 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/recovered.edits] 2024-11-24T02:48:11,617 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0534b8cc899e4406aebb42b7ac44ef75 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/0534b8cc899e4406aebb42b7ac44ef75 2024-11-24T02:48:11,619 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/5d178c9212b24df5aae08c627831357c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/5d178c9212b24df5aae08c627831357c 2024-11-24T02:48:11,621 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/a50409515de345f88584fe311829191f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/a50409515de345f88584fe311829191f 2024-11-24T02:48:11,623 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/c17c354c96694e4a8931a756582526a1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/A/c17c354c96694e4a8931a756582526a1 2024-11-24T02:48:11,626 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/35b7b81246ce44e88697ecb6a71cec06 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/35b7b81246ce44e88697ecb6a71cec06 2024-11-24T02:48:11,628 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/53f375de804648f5941a43e5893da7ed to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/53f375de804648f5941a43e5893da7ed 2024-11-24T02:48:11,629 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/d179246638f24fbcae9e0c473b2c89a4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/d179246638f24fbcae9e0c473b2c89a4 2024-11-24T02:48:11,631 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f8557863d536401d9d2eae97a1d869e0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/B/f8557863d536401d9d2eae97a1d869e0 2024-11-24T02:48:11,634 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/03ed2f0174cd499ea230e816afb26744 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/03ed2f0174cd499ea230e816afb26744 2024-11-24T02:48:11,636 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/44b6fc1ba4934d3d9226816040813f8f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/44b6fc1ba4934d3d9226816040813f8f 2024-11-24T02:48:11,637 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/7b6db45c617b4ea78e3c80783fbd3986 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/7b6db45c617b4ea78e3c80783fbd3986 2024-11-24T02:48:11,639 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/7d9268bbacc94868af9a68155b6f9f7f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/C/7d9268bbacc94868af9a68155b6f9f7f 2024-11-24T02:48:11,643 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/recovered.edits/381.seqid to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140/recovered.edits/381.seqid 2024-11-24T02:48:11,644 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,644 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:48:11,644 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:48:11,646 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T02:48:11,650 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112403763bbe6e0d4bb5ad3ccf8e07f5f399_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112403763bbe6e0d4bb5ad3ccf8e07f5f399_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,651 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404336e5ce486410e8dc5f102ab5d734f_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404336e5ce486410e8dc5f102ab5d734f_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,653 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241d7ba76a31ab45a3a67f5fe423695bbe_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241d7ba76a31ab45a3a67f5fe423695bbe_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,655 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124259a26bc0c3f4a0597cd6ab2a1966136_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124259a26bc0c3f4a0597cd6ab2a1966136_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,656 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242bd65f1cabe24800a24cdf74748918e9_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242bd65f1cabe24800a24cdf74748918e9_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,658 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112446564f506d91435f9aebc62793c94629_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112446564f506d91435f9aebc62793c94629_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,660 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f0b8d229c864240a5ea6dedba2471a8_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f0b8d229c864240a5ea6dedba2471a8_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,662 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112451325bbcc3d947ef8850fdfe8f592f35_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112451325bbcc3d947ef8850fdfe8f592f35_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,664 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456d564375d0245f2808118690fa87b27_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112456d564375d0245f2808118690fa87b27_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,666 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245c0d946ede064b9686a42228f728cf37_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245c0d946ede064b9686a42228f728cf37_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,668 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a5833ed7ab04e888c97e32a8f58135a_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246a5833ed7ab04e888c97e32a8f58135a_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,670 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c188a0ec5bf4bc0bd4fd71eeacabd12_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c188a0ec5bf4bc0bd4fd71eeacabd12_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,672 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112497f1415751074edc959e331e73ef388d_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112497f1415751074edc959e331e73ef388d_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,674 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b12f4457ea754559b038b09fce94a01e_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b12f4457ea754559b038b09fce94a01e_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,676 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b8dc11a2397f45e08a651e9207a1e3b9_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b8dc11a2397f45e08a651e9207a1e3b9_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,678 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bad8f1c4d1f440a2a72ef1238c06f898_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bad8f1c4d1f440a2a72ef1238c06f898_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,680 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bbe1541628e84a65b15f3b9fc9600c30_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bbe1541628e84a65b15f3b9fc9600c30_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,682 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d26ce4165b98451e9ca758cb6683ec3e_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d26ce4165b98451e9ca758cb6683ec3e_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,683 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d34c9be9ea324e9db4debee6586cffe6_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d34c9be9ea324e9db4debee6586cffe6_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,685 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e6de4b8c567a4536944c4425585f5396_859cf8a736ca56969dfab9f223200140 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e6de4b8c567a4536944c4425585f5396_859cf8a736ca56969dfab9f223200140 2024-11-24T02:48:11,686 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:48:11,690 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:11,694 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:48:11,696 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:48:11,697 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:11,697 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:48:11,697 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416491697"}]},"ts":"9223372036854775807"} 2024-11-24T02:48:11,699 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:48:11,699 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 859cf8a736ca56969dfab9f223200140, NAME => 'TestAcidGuarantees,,1732416463208.859cf8a736ca56969dfab9f223200140.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:48:11,700 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:48:11,700 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416491700"}]},"ts":"9223372036854775807"} 2024-11-24T02:48:11,702 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:48:11,702 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:11,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 103 msec 2024-11-24T02:48:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T02:48:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:48:11,717 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:11,726 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 240) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=552 (was 697), ProcessCount=11 (was 11), AvailableMemoryMB=9961 (was 10014) 2024-11-24T02:48:11,735 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=552, ProcessCount=11, AvailableMemoryMB=10120 2024-11-24T02:48:11,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:48:11,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:48:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:11,738 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:48:11,738 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:11,738 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-24T02:48:11,739 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:48:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T02:48:11,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742281_1457 (size=959) 2024-11-24T02:48:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T02:48:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T02:48:12,145 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:48:12,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742282_1458 (size=53) 2024-11-24T02:48:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T02:48:12,553 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:48:12,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5b9e19b406087483d14cf3e721c5b26c, disabling compactions & flushes 2024-11-24T02:48:12,554 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:12,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:12,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. after waiting 0 ms 2024-11-24T02:48:12,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:12,554 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:12,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b9e19b406087483d14cf3e721c5b26c: Waiting for close lock at 1732416492554Disabling compacts and flushes for region at 1732416492554Disabling writes for close at 1732416492554Writing region close event to WAL at 1732416492554Closed at 1732416492554 2024-11-24T02:48:12,555 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:48:12,555 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416492555"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416492555"}]},"ts":"1732416492555"} 2024-11-24T02:48:12,557 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:48:12,558 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:48:12,558 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416492558"}]},"ts":"1732416492558"} 2024-11-24T02:48:12,560 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:48:12,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, ASSIGN}] 2024-11-24T02:48:12,562 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, ASSIGN 2024-11-24T02:48:12,563 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=false 2024-11-24T02:48:12,637 WARN [regionserver/7c69a60bd8f6:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-24T02:48:12,714 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=5b9e19b406087483d14cf3e721c5b26c, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:12,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, ASSIGN because future has completed 2024-11-24T02:48:12,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:48:12,871 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:12,872 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:48:12,872 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,872 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:48:12,873 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,873 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,874 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T02:48:12,877 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9e19b406087483d14cf3e721c5b26c columnFamilyName A 2024-11-24T02:48:12,878 DEBUG [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:12,878 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] regionserver.HStore(327): Store=5b9e19b406087483d14cf3e721c5b26c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:12,878 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,879 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9e19b406087483d14cf3e721c5b26c columnFamilyName B 2024-11-24T02:48:12,879 DEBUG [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:12,879 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] regionserver.HStore(327): Store=5b9e19b406087483d14cf3e721c5b26c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:12,880 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,880 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9e19b406087483d14cf3e721c5b26c columnFamilyName C 2024-11-24T02:48:12,880 DEBUG [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:12,881 INFO [StoreOpener-5b9e19b406087483d14cf3e721c5b26c-1 {}] regionserver.HStore(327): Store=5b9e19b406087483d14cf3e721c5b26c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:12,881 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,881 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,882 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,882 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,882 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,883 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:48:12,883 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,885 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:48:12,885 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 5b9e19b406087483d14cf3e721c5b26c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65850754, jitterRate=-0.018747299909591675}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:48:12,885 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:12,886 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 5b9e19b406087483d14cf3e721c5b26c: Running coprocessor pre-open hook at 1732416492873Writing region info on filesystem at 1732416492873Initializing all the Stores at 1732416492873Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416492874 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416492874Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416492874Cleaning up temporary data from old regions at 1732416492882 (+8 ms)Running coprocessor post-open hooks at 1732416492885 (+3 ms)Region opened successfully at 1732416492886 (+1 ms) 2024-11-24T02:48:12,886 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., pid=120, masterSystemTime=1732416492868 2024-11-24T02:48:12,888 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:12,888 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:12,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=5b9e19b406087483d14cf3e721c5b26c, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:12,890 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:48:12,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-24T02:48:12,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 in 175 msec 2024-11-24T02:48:12,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-24T02:48:12,895 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, ASSIGN in 332 msec 2024-11-24T02:48:12,896 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:48:12,896 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416492896"}]},"ts":"1732416492896"} 2024-11-24T02:48:12,897 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:48:12,898 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:48:12,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-24T02:48:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T02:48:13,898 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:13,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ad78a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,904 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c8a5b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,906 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45197ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,908 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,910 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:48:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,43749,1732416369043 2024-11-24T02:48:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d57a301 2024-11-24T02:48:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:48:13,912 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:48:13,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,913 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b6be8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2bffdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794f3ae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,917 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28320cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,918 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,919 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185f9f2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,920 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7239685b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,922 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627ae65e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3dd14d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3354f79a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0b7f38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,929 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,929 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b76bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,931 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37490, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29de38a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,934 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b369e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,935 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b90866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,938 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,938 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,940 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7d6124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,944 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3146fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,945 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14cc6da4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,947 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:13,947 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:13,947 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260e9b2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:13,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:13,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37542, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:13,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0bbd66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:13,950 DEBUG [Thread-2052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,951 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,951 DEBUG [Thread-2056 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,951 DEBUG [Thread-2058 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,951 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,951 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,952 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,953 DEBUG [Thread-2063 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,953 DEBUG [Thread-2063 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,953 DEBUG [Thread-2054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,953 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:13,954 DEBUG [Thread-2065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,956 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-24T02:48:13,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:48:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,959 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:13,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34326, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,960 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:13,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,964 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:13,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:13,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:13,968 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:13,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:48:13,968 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2] 2024-11-24T02:48:13,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/8e5d7e33f4e84005bd321a4b226fcd6f is 50, key is test_row_0/A:col10/1732416493965/Put/seqid=0 2024-11-24T02:48:13,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742283_1459 (size=14341) 2024-11-24T02:48:14,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/8e5d7e33f4e84005bd321a4b226fcd6f 2024-11-24T02:48:14,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/f17d8e7cb3e14798ac31cded668c8096 is 50, key is test_row_0/B:col10/1732416493965/Put/seqid=0 2024-11-24T02:48:14,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742284_1460 (size=12001) 2024-11-24T02:48:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416554036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416554038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416554039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416554040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416554042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:48:14,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:48:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:14,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:14,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416554157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416554157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416554157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416554157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416554157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:48:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:14,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:48:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416554366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416554378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416554379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416554379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416554379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:48:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:14,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:14,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/f17d8e7cb3e14798ac31cded668c8096 2024-11-24T02:48:14,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/0d528b8a31674907bc4eb92704f0b83a is 50, key is test_row_0/C:col10/1732416493965/Put/seqid=0 2024-11-24T02:48:14,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742285_1461 (size=12001) 2024-11-24T02:48:14,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/0d528b8a31674907bc4eb92704f0b83a 2024-11-24T02:48:14,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/8e5d7e33f4e84005bd321a4b226fcd6f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/8e5d7e33f4e84005bd321a4b226fcd6f 2024-11-24T02:48:14,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/8e5d7e33f4e84005bd321a4b226fcd6f, entries=200, sequenceid=13, filesize=14.0 K 2024-11-24T02:48:14,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/f17d8e7cb3e14798ac31cded668c8096 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f17d8e7cb3e14798ac31cded668c8096 2024-11-24T02:48:14,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f17d8e7cb3e14798ac31cded668c8096, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T02:48:14,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/0d528b8a31674907bc4eb92704f0b83a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0d528b8a31674907bc4eb92704f0b83a 2024-11-24T02:48:14,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0d528b8a31674907bc4eb92704f0b83a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T02:48:14,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5b9e19b406087483d14cf3e721c5b26c in 515ms, sequenceid=13, compaction requested=false 2024-11-24T02:48:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T02:48:14,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:14,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T02:48:14,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:14,575 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:48:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:48:14,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/3f122b0530cb47e0b1147f41e3440c3e is 50, key is test_row_0/A:col10/1732416494036/Put/seqid=0 2024-11-24T02:48:14,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742286_1462 (size=12001) 2024-11-24T02:48:14,607 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/3f122b0530cb47e0b1147f41e3440c3e 2024-11-24T02:48:14,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e6ba069fdd214144b0d16ab57bada65e is 50, key is test_row_0/B:col10/1732416494036/Put/seqid=0 2024-11-24T02:48:14,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742287_1463 (size=12001) 2024-11-24T02:48:14,634 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e6ba069fdd214144b0d16ab57bada65e 2024-11-24T02:48:14,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/27e63cb180a04225ba490ad04ab1bffd is 50, key is test_row_0/C:col10/1732416494036/Put/seqid=0 2024-11-24T02:48:14,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742288_1464 (size=12001) 2024-11-24T02:48:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:14,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416554708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416554709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416554721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416554721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416554722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416554827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416554827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416554836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416554836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:14,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416554847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416555036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416555036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416555056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416555057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416555057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,077 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/27e63cb180a04225ba490ad04ab1bffd 2024-11-24T02:48:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/3f122b0530cb47e0b1147f41e3440c3e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/3f122b0530cb47e0b1147f41e3440c3e 2024-11-24T02:48:15,084 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/3f122b0530cb47e0b1147f41e3440c3e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:48:15,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e6ba069fdd214144b0d16ab57bada65e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6ba069fdd214144b0d16ab57bada65e 2024-11-24T02:48:15,087 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6ba069fdd214144b0d16ab57bada65e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:48:15,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/27e63cb180a04225ba490ad04ab1bffd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/27e63cb180a04225ba490ad04ab1bffd 2024-11-24T02:48:15,090 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/27e63cb180a04225ba490ad04ab1bffd, entries=150, sequenceid=37, filesize=11.7 K 2024-11-24T02:48:15,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for 5b9e19b406087483d14cf3e721c5b26c in 516ms, sequenceid=37, compaction requested=false 2024-11-24T02:48:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:15,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-24T02:48:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-24T02:48:15,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-24T02:48:15,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1320 sec 2024-11-24T02:48:15,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1400 sec 2024-11-24T02:48:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T02:48:15,096 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:15,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-24T02:48:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:48:15,098 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:15,098 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:15,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:48:15,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T02:48:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:15,251 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:48:15,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/c20c9f0f223e49598a013c674938e4ea is 50, key is test_row_0/A:col10/1732416494720/Put/seqid=0 2024-11-24T02:48:15,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742289_1465 (size=12001) 2024-11-24T02:48:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:15,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:15,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416555390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416555398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416555399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416555402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416555405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:48:15,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416555516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416555517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416555517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416555517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416555518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,659 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/c20c9f0f223e49598a013c674938e4ea 2024-11-24T02:48:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c1dfe70fa06a49ccbda33d12710eda05 is 50, key is test_row_0/B:col10/1732416494720/Put/seqid=0 2024-11-24T02:48:15,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742290_1466 (size=12001) 2024-11-24T02:48:15,680 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c1dfe70fa06a49ccbda33d12710eda05 2024-11-24T02:48:15,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/8c8ccf6c5fcb410a9e9f414398d6084c is 50, key is test_row_0/C:col10/1732416494720/Put/seqid=0 2024-11-24T02:48:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742291_1467 (size=12001) 2024-11-24T02:48:15,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/8c8ccf6c5fcb410a9e9f414398d6084c 2024-11-24T02:48:15,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/c20c9f0f223e49598a013c674938e4ea as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c20c9f0f223e49598a013c674938e4ea 2024-11-24T02:48:15,715 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c20c9f0f223e49598a013c674938e4ea, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T02:48:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c1dfe70fa06a49ccbda33d12710eda05 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1dfe70fa06a49ccbda33d12710eda05 2024-11-24T02:48:15,719 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1dfe70fa06a49ccbda33d12710eda05, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T02:48:15,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/8c8ccf6c5fcb410a9e9f414398d6084c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8c8ccf6c5fcb410a9e9f414398d6084c 2024-11-24T02:48:15,723 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8c8ccf6c5fcb410a9e9f414398d6084c, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T02:48:15,723 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5b9e19b406087483d14cf3e721c5b26c in 472ms, sequenceid=51, compaction requested=true 2024-11-24T02:48:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-24T02:48:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-24T02:48:15,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-24T02:48:15,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 627 msec 2024-11-24T02:48:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:48:15,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 629 msec 2024-11-24T02:48:15,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:15,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:15,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T02:48:15,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/22ab3b51e67c41b2b83e95ea76b7f093 is 50, key is test_row_0/A:col10/1732416495728/Put/seqid=0 2024-11-24T02:48:15,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742292_1468 (size=14341) 2024-11-24T02:48:15,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/22ab3b51e67c41b2b83e95ea76b7f093 2024-11-24T02:48:15,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/2ce8c8949c9c4e35a982fb1561869639 is 50, key is test_row_0/B:col10/1732416495728/Put/seqid=0 2024-11-24T02:48:15,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742293_1469 (size=12001) 2024-11-24T02:48:15,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/2ce8c8949c9c4e35a982fb1561869639 2024-11-24T02:48:15,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/1f500698d95b4bedb843792882939648 is 50, key is test_row_0/C:col10/1732416495728/Put/seqid=0 2024-11-24T02:48:15,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742294_1470 (size=12001) 2024-11-24T02:48:15,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416555755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416555766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416555767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416555768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416555769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416555876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416555887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416555887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416555887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416555887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:15,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416556087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416556107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416556107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416556107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416556107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/1f500698d95b4bedb843792882939648 2024-11-24T02:48:16,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/22ab3b51e67c41b2b83e95ea76b7f093 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/22ab3b51e67c41b2b83e95ea76b7f093 2024-11-24T02:48:16,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/22ab3b51e67c41b2b83e95ea76b7f093, entries=200, sequenceid=74, filesize=14.0 K 2024-11-24T02:48:16,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/2ce8c8949c9c4e35a982fb1561869639 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/2ce8c8949c9c4e35a982fb1561869639 2024-11-24T02:48:16,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/2ce8c8949c9c4e35a982fb1561869639, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T02:48:16,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/1f500698d95b4bedb843792882939648 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/1f500698d95b4bedb843792882939648 2024-11-24T02:48:16,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/1f500698d95b4bedb843792882939648, entries=150, sequenceid=74, filesize=11.7 K 2024-11-24T02:48:16,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5b9e19b406087483d14cf3e721c5b26c in 454ms, sequenceid=74, compaction requested=true 2024-11-24T02:48:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:16,183 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:16,183 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:16,184 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:16,184 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:16,184 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:16,184 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:16,184 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:16,184 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:16,184 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f17d8e7cb3e14798ac31cded668c8096, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6ba069fdd214144b0d16ab57bada65e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1dfe70fa06a49ccbda33d12710eda05, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/2ce8c8949c9c4e35a982fb1561869639] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=46.9 K 2024-11-24T02:48:16,184 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/8e5d7e33f4e84005bd321a4b226fcd6f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/3f122b0530cb47e0b1147f41e3440c3e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c20c9f0f223e49598a013c674938e4ea, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/22ab3b51e67c41b2b83e95ea76b7f093] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=51.4 K 2024-11-24T02:48:16,184 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5d7e33f4e84005bd321a4b226fcd6f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416493963 2024-11-24T02:48:16,184 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f17d8e7cb3e14798ac31cded668c8096, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416493965 2024-11-24T02:48:16,185 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e6ba069fdd214144b0d16ab57bada65e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416494036 2024-11-24T02:48:16,185 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f122b0530cb47e0b1147f41e3440c3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416494036 2024-11-24T02:48:16,185 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c1dfe70fa06a49ccbda33d12710eda05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732416494706 2024-11-24T02:48:16,185 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20c9f0f223e49598a013c674938e4ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732416494706 2024-11-24T02:48:16,185 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce8c8949c9c4e35a982fb1561869639, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732416495397 2024-11-24T02:48:16,185 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22ab3b51e67c41b2b83e95ea76b7f093, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732416495397 2024-11-24T02:48:16,191 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:16,191 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:16,192 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/36e38a9d9f884a82a38b7c8f214b32cf is 50, key is test_row_0/A:col10/1732416495728/Put/seqid=0 2024-11-24T02:48:16,192 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/8d6fde395f5043058affa597842bd263 is 50, key is test_row_0/B:col10/1732416495728/Put/seqid=0 2024-11-24T02:48:16,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742296_1472 (size=12139) 2024-11-24T02:48:16,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742295_1471 (size=12139) 2024-11-24T02:48:16,200 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/36e38a9d9f884a82a38b7c8f214b32cf as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/36e38a9d9f884a82a38b7c8f214b32cf 2024-11-24T02:48:16,207 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into 36e38a9d9f884a82a38b7c8f214b32cf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:16,207 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:16,207 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=12, startTime=1732416496183; duration=0sec 2024-11-24T02:48:16,207 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:16,207 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:16,207 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:16,209 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:16,209 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:16,209 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:16,209 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0d528b8a31674907bc4eb92704f0b83a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/27e63cb180a04225ba490ad04ab1bffd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8c8ccf6c5fcb410a9e9f414398d6084c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/1f500698d95b4bedb843792882939648] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=46.9 K 2024-11-24T02:48:16,210 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d528b8a31674907bc4eb92704f0b83a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732416493965 2024-11-24T02:48:16,210 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e63cb180a04225ba490ad04ab1bffd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732416494036 2024-11-24T02:48:16,210 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8ccf6c5fcb410a9e9f414398d6084c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732416494706 2024-11-24T02:48:16,210 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f500698d95b4bedb843792882939648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732416495397 2024-11-24T02:48:16,217 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:16,218 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/ae9772cc41c449e68c3618da07128046 is 50, key is test_row_0/C:col10/1732416495728/Put/seqid=0 2024-11-24T02:48:16,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742297_1473 (size=12139) 2024-11-24T02:48:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T02:48:16,237 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:16,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-24T02:48:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:48:16,239 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:16,240 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:16,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:48:16,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T02:48:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:16,392 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:48:16,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/b1b59ca9da58492e83f6acff6f624032 is 50, key is test_row_0/A:col10/1732416495765/Put/seqid=0 2024-11-24T02:48:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:16,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:16,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742298_1474 (size=12001) 2024-11-24T02:48:16,413 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/b1b59ca9da58492e83f6acff6f624032 2024-11-24T02:48:16,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/fc3130de10c846c9a5a9e11518c7e31e is 50, key is test_row_0/B:col10/1732416495765/Put/seqid=0 2024-11-24T02:48:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742299_1475 (size=12001) 2024-11-24T02:48:16,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416556461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416556472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416556473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416556474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416556476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:48:16,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416556577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416556597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416556597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416556598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,607 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/8d6fde395f5043058affa597842bd263 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/8d6fde395f5043058affa597842bd263 2024-11-24T02:48:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416556599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,621 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into 8d6fde395f5043058affa597842bd263(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:16,621 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:16,621 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=12, startTime=1732416496183; duration=0sec 2024-11-24T02:48:16,621 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:16,621 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:16,633 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/ae9772cc41c449e68c3618da07128046 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/ae9772cc41c449e68c3618da07128046 2024-11-24T02:48:16,637 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into ae9772cc41c449e68c3618da07128046(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:16,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:16,637 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=12, startTime=1732416496183; duration=0sec 2024-11-24T02:48:16,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:16,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:16,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416556797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416556816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416556816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416556817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416556817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,827 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/fc3130de10c846c9a5a9e11518c7e31e 2024-11-24T02:48:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:16,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a8004b6d59a1460b8f983fed459b2c4d is 50, key is test_row_0/C:col10/1732416495765/Put/seqid=0 2024-11-24T02:48:16,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742300_1476 (size=12001) 2024-11-24T02:48:16,840 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a8004b6d59a1460b8f983fed459b2c4d 2024-11-24T02:48:16,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/b1b59ca9da58492e83f6acff6f624032 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/b1b59ca9da58492e83f6acff6f624032 2024-11-24T02:48:16,847 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/b1b59ca9da58492e83f6acff6f624032, entries=150, sequenceid=88, filesize=11.7 K 2024-11-24T02:48:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/fc3130de10c846c9a5a9e11518c7e31e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/fc3130de10c846c9a5a9e11518c7e31e 2024-11-24T02:48:16,851 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/fc3130de10c846c9a5a9e11518c7e31e, entries=150, sequenceid=88, filesize=11.7 K 2024-11-24T02:48:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a8004b6d59a1460b8f983fed459b2c4d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a8004b6d59a1460b8f983fed459b2c4d 2024-11-24T02:48:16,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a8004b6d59a1460b8f983fed459b2c4d, entries=150, sequenceid=88, filesize=11.7 K 2024-11-24T02:48:16,855 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5b9e19b406087483d14cf3e721c5b26c in 463ms, sequenceid=88, compaction requested=false 2024-11-24T02:48:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-24T02:48:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-24T02:48:16,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-24T02:48:16,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 616 msec 2024-11-24T02:48:16,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 622 msec 2024-11-24T02:48:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T02:48:16,868 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:16,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:16,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-24T02:48:16,870 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:48:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:48:16,871 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:16,872 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:16,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:48:17,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T02:48:17,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:17,025 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:48:17,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/590fd4db60fc4136ae75245ed0c8b082 is 50, key is test_row_0/A:col10/1732416496470/Put/seqid=0 2024-11-24T02:48:17,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742301_1477 (size=12001) 2024-11-24T02:48:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:17,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416557137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416557137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416557138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416557142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416557143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:48:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416557247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416557256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416557256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,432 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/590fd4db60fc4136ae75245ed0c8b082 2024-11-24T02:48:17,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/74411638272d4ac4ba9def90859f72b5 is 50, key is test_row_0/B:col10/1732416496470/Put/seqid=0 2024-11-24T02:48:17,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742302_1478 (size=12001) 2024-11-24T02:48:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416557457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416557466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416557466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:48:17,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416557646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416557657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416557767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416557777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416557777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:17,842 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/74411638272d4ac4ba9def90859f72b5 2024-11-24T02:48:17,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/17e536830159448da66ebb4363e37edb is 50, key is test_row_0/C:col10/1732416496470/Put/seqid=0 2024-11-24T02:48:17,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742303_1479 (size=12001) 2024-11-24T02:48:17,854 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/17e536830159448da66ebb4363e37edb 2024-11-24T02:48:17,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/590fd4db60fc4136ae75245ed0c8b082 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/590fd4db60fc4136ae75245ed0c8b082 2024-11-24T02:48:17,867 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/590fd4db60fc4136ae75245ed0c8b082, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:48:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/74411638272d4ac4ba9def90859f72b5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/74411638272d4ac4ba9def90859f72b5 2024-11-24T02:48:17,873 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/74411638272d4ac4ba9def90859f72b5, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:48:17,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/17e536830159448da66ebb4363e37edb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/17e536830159448da66ebb4363e37edb 2024-11-24T02:48:17,879 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/17e536830159448da66ebb4363e37edb, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:48:17,884 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5b9e19b406087483d14cf3e721c5b26c in 859ms, sequenceid=114, compaction requested=true 2024-11-24T02:48:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-24T02:48:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-24T02:48:17,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-24T02:48:17,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-24T02:48:17,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0190 sec 2024-11-24T02:48:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T02:48:18,007 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:18,008 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-24T02:48:18,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:48:18,011 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:18,012 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:18,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:48:18,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T02:48:18,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:18,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:48:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/4610286483ae4516ab53735530c63575 is 50, key is test_row_0/A:col10/1732416497142/Put/seqid=0 2024-11-24T02:48:18,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742304_1480 (size=12001) 2024-11-24T02:48:18,178 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/4610286483ae4516ab53735530c63575 2024-11-24T02:48:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/6a8dc8418ea84f909952a0f671609e5c is 50, key is test_row_0/B:col10/1732416497142/Put/seqid=0 2024-11-24T02:48:18,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742305_1481 (size=12001) 2024-11-24T02:48:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:18,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:48:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416558379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416558390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416558390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416558496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416558517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416558517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:48:18,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/6a8dc8418ea84f909952a0f671609e5c 2024-11-24T02:48:18,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/06ab6499fe9e4c40917ee5177000d34c is 50, key is test_row_0/C:col10/1732416497142/Put/seqid=0 2024-11-24T02:48:18,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742306_1482 (size=12001) 2024-11-24T02:48:18,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416558666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,674 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416558676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,685 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416558716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416558727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416558727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416559026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416559046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416559046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,066 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/06ab6499fe9e4c40917ee5177000d34c 2024-11-24T02:48:19,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/4610286483ae4516ab53735530c63575 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4610286483ae4516ab53735530c63575 2024-11-24T02:48:19,072 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4610286483ae4516ab53735530c63575, entries=150, sequenceid=126, filesize=11.7 K 2024-11-24T02:48:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/6a8dc8418ea84f909952a0f671609e5c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/6a8dc8418ea84f909952a0f671609e5c 2024-11-24T02:48:19,076 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/6a8dc8418ea84f909952a0f671609e5c, entries=150, sequenceid=126, filesize=11.7 K 2024-11-24T02:48:19,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/06ab6499fe9e4c40917ee5177000d34c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/06ab6499fe9e4c40917ee5177000d34c 2024-11-24T02:48:19,080 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/06ab6499fe9e4c40917ee5177000d34c, entries=150, sequenceid=126, filesize=11.7 K 2024-11-24T02:48:19,081 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5b9e19b406087483d14cf3e721c5b26c in 917ms, sequenceid=126, compaction requested=true 2024-11-24T02:48:19,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:19,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:19,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-24T02:48:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-24T02:48:19,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-24T02:48:19,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0700 sec 2024-11-24T02:48:19,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0760 sec 2024-11-24T02:48:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T02:48:19,147 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:19,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-24T02:48:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:48:19,148 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:19,149 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:19,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:48:19,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T02:48:19,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:19,301 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:48:19,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/2b3d97cb5a544308adfd6415ec93d152 is 50, key is test_row_0/A:col10/1732416498388/Put/seqid=0 2024-11-24T02:48:19,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742307_1483 (size=12151) 2024-11-24T02:48:19,320 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/2b3d97cb5a544308adfd6415ec93d152 2024-11-24T02:48:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/4860fec3fe1b4d33b646b952a53425fa is 50, key is test_row_0/B:col10/1732416498388/Put/seqid=0 2024-11-24T02:48:19,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742308_1484 (size=12151) 2024-11-24T02:48:19,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:48:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:19,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:19,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416559581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416559589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416559590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416559706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416559716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416559717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,735 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/4860fec3fe1b4d33b646b952a53425fa 2024-11-24T02:48:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/d55081b9b36f44f9b08c5c3204d31f71 is 50, key is test_row_0/C:col10/1732416498388/Put/seqid=0 2024-11-24T02:48:19,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742309_1485 (size=12151) 2024-11-24T02:48:19,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:48:19,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416559926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416559927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416559927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,150 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/d55081b9b36f44f9b08c5c3204d31f71 2024-11-24T02:48:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/2b3d97cb5a544308adfd6415ec93d152 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2b3d97cb5a544308adfd6415ec93d152 2024-11-24T02:48:20,164 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2b3d97cb5a544308adfd6415ec93d152, entries=150, sequenceid=150, filesize=11.9 K 2024-11-24T02:48:20,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/4860fec3fe1b4d33b646b952a53425fa as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/4860fec3fe1b4d33b646b952a53425fa 2024-11-24T02:48:20,169 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/4860fec3fe1b4d33b646b952a53425fa, entries=150, sequenceid=150, filesize=11.9 K 2024-11-24T02:48:20,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/d55081b9b36f44f9b08c5c3204d31f71 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/d55081b9b36f44f9b08c5c3204d31f71 2024-11-24T02:48:20,176 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/d55081b9b36f44f9b08c5c3204d31f71, entries=150, sequenceid=150, filesize=11.9 K 2024-11-24T02:48:20,178 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5b9e19b406087483d14cf3e721c5b26c in 877ms, sequenceid=150, compaction requested=true 2024-11-24T02:48:20,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:20,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-24T02:48:20,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-24T02:48:20,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-24T02:48:20,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0310 sec 2024-11-24T02:48:20,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0360 sec 2024-11-24T02:48:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:20,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:20,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:48:20,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/a1dc2949116c4c9f920e4d38a61dd12a is 50, key is test_row_0/A:col10/1732416499589/Put/seqid=0 2024-11-24T02:48:20,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742310_1486 (size=14541) 2024-11-24T02:48:20,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/a1dc2949116c4c9f920e4d38a61dd12a 2024-11-24T02:48:20,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/a82962771b2b48f8b79d3c5b9f231eff is 50, key is test_row_0/B:col10/1732416499589/Put/seqid=0 2024-11-24T02:48:20,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742311_1487 (size=12151) 2024-11-24T02:48:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T02:48:20,287 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:20,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-24T02:48:20,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:48:20,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:20,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:20,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416560337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416560343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416560344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:48:20,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:48:20,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:20,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416560456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416560457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416560467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:48:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:48:20,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/a82962771b2b48f8b79d3c5b9f231eff 2024-11-24T02:48:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416560676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416560677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416560677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/60c5d036922e4e30b4b14daaaa3cfa19 is 50, key is test_row_0/C:col10/1732416499589/Put/seqid=0 2024-11-24T02:48:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742312_1488 (size=12151) 2024-11-24T02:48:20,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416560697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,709 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416560707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:20,724 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:20,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:48:20,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:20,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:48:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:20,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:20,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:48:21,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416560996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416560997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416560997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:48:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:21,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:21,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:21,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:21,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/60c5d036922e4e30b4b14daaaa3cfa19 2024-11-24T02:48:21,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/a1dc2949116c4c9f920e4d38a61dd12a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/a1dc2949116c4c9f920e4d38a61dd12a 2024-11-24T02:48:21,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/a1dc2949116c4c9f920e4d38a61dd12a, entries=200, sequenceid=163, filesize=14.2 K 2024-11-24T02:48:21,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/a82962771b2b48f8b79d3c5b9f231eff as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a82962771b2b48f8b79d3c5b9f231eff 2024-11-24T02:48:21,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a82962771b2b48f8b79d3c5b9f231eff, entries=150, sequenceid=163, filesize=11.9 K 2024-11-24T02:48:21,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/60c5d036922e4e30b4b14daaaa3cfa19 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/60c5d036922e4e30b4b14daaaa3cfa19 2024-11-24T02:48:21,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/60c5d036922e4e30b4b14daaaa3cfa19, entries=150, sequenceid=163, filesize=11.9 K 2024-11-24T02:48:21,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5b9e19b406087483d14cf3e721c5b26c in 871ms, sequenceid=163, compaction requested=true 2024-11-24T02:48:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:21,108 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-24T02:48:21,108 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-24T02:48:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:21,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:21,110 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74834 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-24T02:48:21,110 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72444 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-24T02:48:21,110 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:21,110 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:21,110 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:21,110 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:21,110 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/36e38a9d9f884a82a38b7c8f214b32cf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/b1b59ca9da58492e83f6acff6f624032, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/590fd4db60fc4136ae75245ed0c8b082, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4610286483ae4516ab53735530c63575, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2b3d97cb5a544308adfd6415ec93d152, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/a1dc2949116c4c9f920e4d38a61dd12a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=73.1 K 2024-11-24T02:48:21,110 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/8d6fde395f5043058affa597842bd263, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/fc3130de10c846c9a5a9e11518c7e31e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/74411638272d4ac4ba9def90859f72b5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/6a8dc8418ea84f909952a0f671609e5c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/4860fec3fe1b4d33b646b952a53425fa, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a82962771b2b48f8b79d3c5b9f231eff] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=70.7 K 2024-11-24T02:48:21,111 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36e38a9d9f884a82a38b7c8f214b32cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732416495397 2024-11-24T02:48:21,111 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d6fde395f5043058affa597842bd263, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732416495397 2024-11-24T02:48:21,111 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3130de10c846c9a5a9e11518c7e31e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416495748 2024-11-24T02:48:21,111 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1b59ca9da58492e83f6acff6f624032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416495748 2024-11-24T02:48:21,111 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 74411638272d4ac4ba9def90859f72b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416496470 2024-11-24T02:48:21,111 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 590fd4db60fc4136ae75245ed0c8b082, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416496470 2024-11-24T02:48:21,112 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a8dc8418ea84f909952a0f671609e5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732416497128 2024-11-24T02:48:21,112 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4610286483ae4516ab53735530c63575, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732416497128 2024-11-24T02:48:21,112 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4860fec3fe1b4d33b646b952a53425fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732416498377 2024-11-24T02:48:21,112 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3d97cb5a544308adfd6415ec93d152, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732416498377 2024-11-24T02:48:21,112 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting a82962771b2b48f8b79d3c5b9f231eff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732416499586 2024-11-24T02:48:21,112 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1dc2949116c4c9f920e4d38a61dd12a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732416499578 2024-11-24T02:48:21,133 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#410 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:21,133 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/641022970b62450b9ca753c6ad745d93 is 50, key is test_row_0/A:col10/1732416499589/Put/seqid=0 2024-11-24T02:48:21,135 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#411 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:21,135 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/1ecba8ba26f6472d9e151ac8aad85930 is 50, key is test_row_0/B:col10/1732416499589/Put/seqid=0 2024-11-24T02:48:21,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742314_1490 (size=12493) 2024-11-24T02:48:21,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742313_1489 (size=12493) 2024-11-24T02:48:21,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T02:48:21,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:21,207 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T02:48:21,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/2d33521384614614bf0bf197fb0a52e6 is 50, key is test_row_0/A:col10/1732416500319/Put/seqid=0 2024-11-24T02:48:21,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742315_1491 (size=12151) 2024-11-24T02:48:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:48:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:21,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:21,563 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/1ecba8ba26f6472d9e151ac8aad85930 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1ecba8ba26f6472d9e151ac8aad85930 2024-11-24T02:48:21,564 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/641022970b62450b9ca753c6ad745d93 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/641022970b62450b9ca753c6ad745d93 2024-11-24T02:48:21,569 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into 641022970b62450b9ca753c6ad745d93(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:21,569 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:21,569 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=10, startTime=1732416501108; duration=0sec 2024-11-24T02:48:21,569 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:21,569 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:21,569 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-24T02:48:21,570 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into 1ecba8ba26f6472d9e151ac8aad85930(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:21,570 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:21,570 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=10, startTime=1732416501108; duration=0sec 2024-11-24T02:48:21,570 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:21,570 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:21,571 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72444 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-24T02:48:21,571 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:21,571 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:21,571 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/ae9772cc41c449e68c3618da07128046, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a8004b6d59a1460b8f983fed459b2c4d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/17e536830159448da66ebb4363e37edb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/06ab6499fe9e4c40917ee5177000d34c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/d55081b9b36f44f9b08c5c3204d31f71, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/60c5d036922e4e30b4b14daaaa3cfa19] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=70.7 K 2024-11-24T02:48:21,571 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae9772cc41c449e68c3618da07128046, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732416495397 2024-11-24T02:48:21,571 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8004b6d59a1460b8f983fed459b2c4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732416495748 2024-11-24T02:48:21,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17e536830159448da66ebb4363e37edb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416496470 2024-11-24T02:48:21,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06ab6499fe9e4c40917ee5177000d34c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732416497128 2024-11-24T02:48:21,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55081b9b36f44f9b08c5c3204d31f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732416498377 2024-11-24T02:48:21,572 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c5d036922e4e30b4b14daaaa3cfa19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732416499586 2024-11-24T02:48:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416561568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416561571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416561577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,596 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#413 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:21,596 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/11a47b67c4c448808ae8b5caf422d8f5 is 50, key is test_row_0/C:col10/1732416499589/Put/seqid=0 2024-11-24T02:48:21,615 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/2d33521384614614bf0bf197fb0a52e6 2024-11-24T02:48:21,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742316_1492 (size=12493) 2024-11-24T02:48:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/f76293c8bd59494eafc175c80217f1dd is 50, key is test_row_0/B:col10/1732416500319/Put/seqid=0 2024-11-24T02:48:21,641 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/11a47b67c4c448808ae8b5caf422d8f5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/11a47b67c4c448808ae8b5caf422d8f5 2024-11-24T02:48:21,646 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into 11a47b67c4c448808ae8b5caf422d8f5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:21,646 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:21,646 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=10, startTime=1732416501108; duration=0sec 2024-11-24T02:48:21,646 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:21,646 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:21,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742317_1493 (size=12151) 2024-11-24T02:48:21,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416561697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416561697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416561698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416561907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416561907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416561907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,073 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/f76293c8bd59494eafc175c80217f1dd 2024-11-24T02:48:22,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/59b56f085dcb4e31bd2aca390fe9c112 is 50, key is test_row_0/C:col10/1732416500319/Put/seqid=0 2024-11-24T02:48:22,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742318_1494 (size=12151) 2024-11-24T02:48:22,111 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/59b56f085dcb4e31bd2aca390fe9c112 2024-11-24T02:48:22,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/2d33521384614614bf0bf197fb0a52e6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2d33521384614614bf0bf197fb0a52e6 2024-11-24T02:48:22,124 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2d33521384614614bf0bf197fb0a52e6, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T02:48:22,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/f76293c8bd59494eafc175c80217f1dd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f76293c8bd59494eafc175c80217f1dd 2024-11-24T02:48:22,129 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f76293c8bd59494eafc175c80217f1dd, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T02:48:22,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/59b56f085dcb4e31bd2aca390fe9c112 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/59b56f085dcb4e31bd2aca390fe9c112 2024-11-24T02:48:22,133 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/59b56f085dcb4e31bd2aca390fe9c112, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T02:48:22,134 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5b9e19b406087483d14cf3e721c5b26c in 927ms, sequenceid=186, compaction requested=false 2024-11-24T02:48:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-24T02:48:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-24T02:48:22,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-24T02:48:22,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8470 sec 2024-11-24T02:48:22,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.8510 sec 2024-11-24T02:48:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:22,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:22,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:48:22,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/9d405acf2161459a98bbd4039cb46aa2 is 50, key is test_row_0/A:col10/1732416501574/Put/seqid=0 2024-11-24T02:48:22,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742319_1495 (size=14541) 2024-11-24T02:48:22,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416562283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416562297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416562298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416562407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416562416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416562416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T02:48:22,437 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:22,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-24T02:48:22,443 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:22,444 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:48:22,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:48:22,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:48:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:22,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416562617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/9d405acf2161459a98bbd4039cb46aa2 2024-11-24T02:48:22,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/df291f72a492431c9d37193c81bfd29d is 50, key is test_row_0/B:col10/1732416501574/Put/seqid=0 2024-11-24T02:48:22,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416562637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416562637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742320_1496 (size=12151) 2024-11-24T02:48:22,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:48:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:22,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:48:22,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:48:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:22,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:22,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416562937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416562947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416562957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/df291f72a492431c9d37193c81bfd29d 2024-11-24T02:48:23,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:48:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:23,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:23,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:23,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/f814413559eb4fa193ae75ed239ecaca is 50, key is test_row_0/C:col10/1732416501574/Put/seqid=0 2024-11-24T02:48:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:48:23,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742321_1497 (size=12151) 2024-11-24T02:48:23,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/f814413559eb4fa193ae75ed239ecaca 2024-11-24T02:48:23,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/9d405acf2161459a98bbd4039cb46aa2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/9d405acf2161459a98bbd4039cb46aa2 2024-11-24T02:48:23,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/9d405acf2161459a98bbd4039cb46aa2, entries=200, sequenceid=203, filesize=14.2 K 2024-11-24T02:48:23,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/df291f72a492431c9d37193c81bfd29d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/df291f72a492431c9d37193c81bfd29d 2024-11-24T02:48:23,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/df291f72a492431c9d37193c81bfd29d, entries=150, sequenceid=203, filesize=11.9 K 2024-11-24T02:48:23,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/f814413559eb4fa193ae75ed239ecaca as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f814413559eb4fa193ae75ed239ecaca 2024-11-24T02:48:23,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f814413559eb4fa193ae75ed239ecaca, entries=150, sequenceid=203, filesize=11.9 K 2024-11-24T02:48:23,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 5b9e19b406087483d14cf3e721c5b26c in 873ms, sequenceid=203, compaction requested=true 2024-11-24T02:48:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:23,096 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:23,096 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:23,096 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:23,097 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:23,097 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:23,097 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1ecba8ba26f6472d9e151ac8aad85930, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f76293c8bd59494eafc175c80217f1dd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/df291f72a492431c9d37193c81bfd29d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=35.9 K 2024-11-24T02:48:23,097 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/641022970b62450b9ca753c6ad745d93, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2d33521384614614bf0bf197fb0a52e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/9d405acf2161459a98bbd4039cb46aa2] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=38.3 K 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ecba8ba26f6472d9e151ac8aad85930, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732416499586 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 641022970b62450b9ca753c6ad745d93, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732416499586 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f76293c8bd59494eafc175c80217f1dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732416500319 2024-11-24T02:48:23,097 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d33521384614614bf0bf197fb0a52e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732416500319 2024-11-24T02:48:23,098 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting df291f72a492431c9d37193c81bfd29d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732416501531 2024-11-24T02:48:23,098 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d405acf2161459a98bbd4039cb46aa2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732416501531 2024-11-24T02:48:23,105 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:23,105 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/99d305c7b8404436b053afb310d889cb is 50, key is test_row_0/A:col10/1732416501574/Put/seqid=0 2024-11-24T02:48:23,107 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:23,107 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e6da7da3987941688fbe29f0dc515e1b is 50, key is test_row_0/B:col10/1732416501574/Put/seqid=0 2024-11-24T02:48:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742322_1498 (size=12595) 2024-11-24T02:48:23,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742323_1499 (size=12595) 2024-11-24T02:48:23,115 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e6da7da3987941688fbe29f0dc515e1b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6da7da3987941688fbe29f0dc515e1b 2024-11-24T02:48:23,121 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into e6da7da3987941688fbe29f0dc515e1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:23,121 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:23,121 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=13, startTime=1732416503096; duration=0sec 2024-11-24T02:48:23,122 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:23,122 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:23,122 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:23,122 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:23,123 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:23,123 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:23,123 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/11a47b67c4c448808ae8b5caf422d8f5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/59b56f085dcb4e31bd2aca390fe9c112, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f814413559eb4fa193ae75ed239ecaca] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=35.9 K 2024-11-24T02:48:23,123 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a47b67c4c448808ae8b5caf422d8f5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1732416499586 2024-11-24T02:48:23,123 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 59b56f085dcb4e31bd2aca390fe9c112, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732416500319 2024-11-24T02:48:23,124 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting f814413559eb4fa193ae75ed239ecaca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732416501531 2024-11-24T02:48:23,133 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#421 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:23,134 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a6e88666efda4871863babe485c810cd is 50, key is test_row_0/C:col10/1732416501574/Put/seqid=0 2024-11-24T02:48:23,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742324_1500 (size=12595) 2024-11-24T02:48:23,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T02:48:23,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:23,212 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T02:48:23,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/eb480f627eec4039a4ee65a19c4da7dc is 50, key is test_row_0/A:col10/1732416502295/Put/seqid=0 2024-11-24T02:48:23,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742325_1501 (size=12151) 2024-11-24T02:48:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:23,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:23,513 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/99d305c7b8404436b053afb310d889cb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/99d305c7b8404436b053afb310d889cb 2024-11-24T02:48:23,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416563500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416563503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416563504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,518 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into 99d305c7b8404436b053afb310d889cb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:23,518 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:23,518 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=13, startTime=1732416503096; duration=0sec 2024-11-24T02:48:23,519 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:23,519 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:48:23,576 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a6e88666efda4871863babe485c810cd as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a6e88666efda4871863babe485c810cd 2024-11-24T02:48:23,583 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into a6e88666efda4871863babe485c810cd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:23,583 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:23,583 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=13, startTime=1732416503096; duration=0sec 2024-11-24T02:48:23,583 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:23,583 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:23,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416563617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416563617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416563626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,648 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/eb480f627eec4039a4ee65a19c4da7dc 2024-11-24T02:48:23,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e1b89a5b9fa34b27ad60d7b0c448dd99 is 50, key is test_row_0/B:col10/1732416502295/Put/seqid=0 2024-11-24T02:48:23,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742326_1502 (size=12151) 2024-11-24T02:48:23,678 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e1b89a5b9fa34b27ad60d7b0c448dd99 2024-11-24T02:48:23,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/8f9dc19c231145ccbfef317bc9edf23e is 50, key is test_row_0/C:col10/1732416502295/Put/seqid=0 2024-11-24T02:48:23,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742327_1503 (size=12151) 2024-11-24T02:48:23,689 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/8f9dc19c231145ccbfef317bc9edf23e 2024-11-24T02:48:23,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/eb480f627eec4039a4ee65a19c4da7dc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/eb480f627eec4039a4ee65a19c4da7dc 2024-11-24T02:48:23,698 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/eb480f627eec4039a4ee65a19c4da7dc, entries=150, sequenceid=227, filesize=11.9 K 2024-11-24T02:48:23,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e1b89a5b9fa34b27ad60d7b0c448dd99 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e1b89a5b9fa34b27ad60d7b0c448dd99 2024-11-24T02:48:23,706 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e1b89a5b9fa34b27ad60d7b0c448dd99, entries=150, sequenceid=227, filesize=11.9 K 2024-11-24T02:48:23,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/8f9dc19c231145ccbfef317bc9edf23e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8f9dc19c231145ccbfef317bc9edf23e 2024-11-24T02:48:23,713 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8f9dc19c231145ccbfef317bc9edf23e, entries=150, sequenceid=227, filesize=11.9 K 2024-11-24T02:48:23,714 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5b9e19b406087483d14cf3e721c5b26c in 502ms, sequenceid=227, compaction requested=false 2024-11-24T02:48:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:23,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-24T02:48:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-24T02:48:23,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-24T02:48:23,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-11-24T02:48:23,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.2830 sec 2024-11-24T02:48:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:23,842 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:23,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:48:23,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f184dd6548ec4335b77f40aaf203414e is 50, key is test_row_0/A:col10/1732416503502/Put/seqid=0 2024-11-24T02:48:23,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742328_1504 (size=14541) 2024-11-24T02:48:23,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416563915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416563925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:23,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416563927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416564036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416564036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416564048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f184dd6548ec4335b77f40aaf203414e 2024-11-24T02:48:24,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e490404198a949f18b382c05b3b9f6ba is 50, key is test_row_0/B:col10/1732416503502/Put/seqid=0 2024-11-24T02:48:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416564257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416564257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416564257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742329_1505 (size=12151) 2024-11-24T02:48:24,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e490404198a949f18b382c05b3b9f6ba 2024-11-24T02:48:24,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/f227cc0f24644467a3caf3e311fc9188 is 50, key is test_row_0/C:col10/1732416503502/Put/seqid=0 2024-11-24T02:48:24,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742330_1506 (size=12151) 2024-11-24T02:48:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416564577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416564577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T02:48:24,586 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:24,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416564577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,587 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-24T02:48:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:48:24,589 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:24,589 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:24,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:24,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/f227cc0f24644467a3caf3e311fc9188 2024-11-24T02:48:24,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f184dd6548ec4335b77f40aaf203414e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f184dd6548ec4335b77f40aaf203414e 2024-11-24T02:48:24,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f184dd6548ec4335b77f40aaf203414e, entries=200, sequenceid=243, filesize=14.2 K 2024-11-24T02:48:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:48:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/e490404198a949f18b382c05b3b9f6ba as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e490404198a949f18b382c05b3b9f6ba 2024-11-24T02:48:24,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e490404198a949f18b382c05b3b9f6ba, entries=150, sequenceid=243, filesize=11.9 K 2024-11-24T02:48:24,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/f227cc0f24644467a3caf3e311fc9188 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f227cc0f24644467a3caf3e311fc9188 2024-11-24T02:48:24,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f227cc0f24644467a3caf3e311fc9188, entries=150, sequenceid=243, filesize=11.9 K 2024-11-24T02:48:24,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5b9e19b406087483d14cf3e721c5b26c in 863ms, sequenceid=243, compaction requested=true 2024-11-24T02:48:24,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:24,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:24,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:24,705 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:24,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:24,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:24,705 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:24,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:24,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:24,706 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:24,706 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:24,706 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:24,706 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/99d305c7b8404436b053afb310d889cb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/eb480f627eec4039a4ee65a19c4da7dc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f184dd6548ec4335b77f40aaf203414e] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=38.4 K 2024-11-24T02:48:24,706 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:24,706 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:24,706 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:24,706 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6da7da3987941688fbe29f0dc515e1b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e1b89a5b9fa34b27ad60d7b0c448dd99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e490404198a949f18b382c05b3b9f6ba] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.0 K 2024-11-24T02:48:24,707 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d305c7b8404436b053afb310d889cb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732416501531 2024-11-24T02:48:24,707 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e6da7da3987941688fbe29f0dc515e1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732416501531 2024-11-24T02:48:24,707 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e1b89a5b9fa34b27ad60d7b0c448dd99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416502268 2024-11-24T02:48:24,707 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb480f627eec4039a4ee65a19c4da7dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416502268 2024-11-24T02:48:24,707 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting e490404198a949f18b382c05b3b9f6ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732416503474 2024-11-24T02:48:24,707 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f184dd6548ec4335b77f40aaf203414e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732416503474 2024-11-24T02:48:24,715 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:24,715 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/ac53f8f794f6437e84cc046e645d6ff8 is 50, key is test_row_0/A:col10/1732416503502/Put/seqid=0 2024-11-24T02:48:24,725 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:24,726 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/faf96641872b4307bf6641b80d0de1c1 is 50, key is test_row_0/B:col10/1732416503502/Put/seqid=0 2024-11-24T02:48:24,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742331_1507 (size=12697) 2024-11-24T02:48:24,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742332_1508 (size=12697) 2024-11-24T02:48:24,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T02:48:24,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:24,742 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:48:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:24,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:24,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f62de0daf3c646a0afed0f4c5da6c19e is 50, key is test_row_0/A:col10/1732416503913/Put/seqid=0 2024-11-24T02:48:24,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742333_1509 (size=12301) 2024-11-24T02:48:24,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416564789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416564796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:48:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416564907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416564907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416565087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416565087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416565096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,130 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/ac53f8f794f6437e84cc046e645d6ff8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/ac53f8f794f6437e84cc046e645d6ff8 2024-11-24T02:48:25,135 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into ac53f8f794f6437e84cc046e645d6ff8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:25,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,135 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:25,135 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=13, startTime=1732416504705; duration=0sec 2024-11-24T02:48:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416565127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,135 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:25,135 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:25,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416565127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,135 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,138 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:25,138 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:25,138 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:25,138 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a6e88666efda4871863babe485c810cd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8f9dc19c231145ccbfef317bc9edf23e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f227cc0f24644467a3caf3e311fc9188] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.0 K 2024-11-24T02:48:25,138 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/faf96641872b4307bf6641b80d0de1c1 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/faf96641872b4307bf6641b80d0de1c1 2024-11-24T02:48:25,138 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6e88666efda4871863babe485c810cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732416501531 2024-11-24T02:48:25,139 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f9dc19c231145ccbfef317bc9edf23e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732416502268 2024-11-24T02:48:25,139 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f227cc0f24644467a3caf3e311fc9188, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732416503474 2024-11-24T02:48:25,144 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into faf96641872b4307bf6641b80d0de1c1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:25,144 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:25,144 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=13, startTime=1732416504705; duration=0sec 2024-11-24T02:48:25,144 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:25,144 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:25,148 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:25,149 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/4702861d11eb4bd6918ec81c74cb6c47 is 50, key is test_row_0/C:col10/1732416503502/Put/seqid=0 2024-11-24T02:48:25,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742334_1510 (size=12697) 2024-11-24T02:48:25,153 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f62de0daf3c646a0afed0f4c5da6c19e 2024-11-24T02:48:25,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c1a24038814a45d9a4f233295b2494b8 is 50, key is test_row_0/B:col10/1732416503913/Put/seqid=0 2024-11-24T02:48:25,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742335_1511 (size=12301) 2024-11-24T02:48:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:48:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416565447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416565447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,557 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/4702861d11eb4bd6918ec81c74cb6c47 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/4702861d11eb4bd6918ec81c74cb6c47 2024-11-24T02:48:25,564 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into 4702861d11eb4bd6918ec81c74cb6c47(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:25,564 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:25,564 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=13, startTime=1732416504705; duration=0sec 2024-11-24T02:48:25,564 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:25,564 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:25,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c1a24038814a45d9a4f233295b2494b8 2024-11-24T02:48:25,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/308b7ee89f8c472dbd0e3dea8bf9fa77 is 50, key is test_row_0/C:col10/1732416503913/Put/seqid=0 2024-11-24T02:48:25,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742336_1512 (size=12301) 2024-11-24T02:48:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:48:25,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416565976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416565976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,037 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/308b7ee89f8c472dbd0e3dea8bf9fa77 2024-11-24T02:48:26,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f62de0daf3c646a0afed0f4c5da6c19e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f62de0daf3c646a0afed0f4c5da6c19e 2024-11-24T02:48:26,043 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f62de0daf3c646a0afed0f4c5da6c19e, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T02:48:26,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c1a24038814a45d9a4f233295b2494b8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1a24038814a45d9a4f233295b2494b8 2024-11-24T02:48:26,046 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1a24038814a45d9a4f233295b2494b8, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T02:48:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/308b7ee89f8c472dbd0e3dea8bf9fa77 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/308b7ee89f8c472dbd0e3dea8bf9fa77 2024-11-24T02:48:26,049 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/308b7ee89f8c472dbd0e3dea8bf9fa77, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T02:48:26,050 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5b9e19b406087483d14cf3e721c5b26c in 1308ms, sequenceid=265, compaction requested=false 2024-11-24T02:48:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:26,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-24T02:48:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-24T02:48:26,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-24T02:48:26,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4620 sec 2024-11-24T02:48:26,054 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.4660 sec 2024-11-24T02:48:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:26,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:26,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T02:48:26,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/47cbc1536c3240f19f7b6836c717142b is 50, key is test_row_0/A:col10/1732416506112/Put/seqid=0 2024-11-24T02:48:26,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742337_1513 (size=14741) 2024-11-24T02:48:26,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/47cbc1536c3240f19f7b6836c717142b 2024-11-24T02:48:26,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/7a9975763a7042e4b2902d9001899be9 is 50, key is test_row_0/B:col10/1732416506112/Put/seqid=0 2024-11-24T02:48:26,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742338_1514 (size=12301) 2024-11-24T02:48:26,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416566168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416566169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416566181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416566287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416566287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416566296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416566507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416566507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416566507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/7a9975763a7042e4b2902d9001899be9 2024-11-24T02:48:26,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/cbc8c2fd47da4b3fa44070b76da5be61 is 50, key is test_row_0/C:col10/1732416506112/Put/seqid=0 2024-11-24T02:48:26,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742339_1515 (size=12301) 2024-11-24T02:48:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T02:48:26,737 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:26,738 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-24T02:48:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T02:48:26,739 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:26,740 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:26,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416566827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416566827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416566827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:26,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T02:48:26,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:26,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:26,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:26,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/cbc8c2fd47da4b3fa44070b76da5be61 2024-11-24T02:48:26,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/47cbc1536c3240f19f7b6836c717142b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/47cbc1536c3240f19f7b6836c717142b 2024-11-24T02:48:26,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/47cbc1536c3240f19f7b6836c717142b, entries=200, sequenceid=283, filesize=14.4 K 2024-11-24T02:48:26,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/7a9975763a7042e4b2902d9001899be9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/7a9975763a7042e4b2902d9001899be9 2024-11-24T02:48:26,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/7a9975763a7042e4b2902d9001899be9, entries=150, sequenceid=283, filesize=12.0 K 2024-11-24T02:48:26,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/cbc8c2fd47da4b3fa44070b76da5be61 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/cbc8c2fd47da4b3fa44070b76da5be61 2024-11-24T02:48:26,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/cbc8c2fd47da4b3fa44070b76da5be61, entries=150, sequenceid=283, filesize=12.0 K 2024-11-24T02:48:26,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5b9e19b406087483d14cf3e721c5b26c in 851ms, sequenceid=283, compaction requested=true 2024-11-24T02:48:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:26,966 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:26,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T02:48:26,967 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:26,967 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:26,967 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:26,967 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/ac53f8f794f6437e84cc046e645d6ff8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f62de0daf3c646a0afed0f4c5da6c19e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/47cbc1536c3240f19f7b6836c717142b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=38.8 K 2024-11-24T02:48:26,967 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac53f8f794f6437e84cc046e645d6ff8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732416503474 2024-11-24T02:48:26,967 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f62de0daf3c646a0afed0f4c5da6c19e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732416503911 2024-11-24T02:48:26,968 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47cbc1536c3240f19f7b6836c717142b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732416504786 2024-11-24T02:48:26,968 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:26,969 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:26,969 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:26,969 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:26,969 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/faf96641872b4307bf6641b80d0de1c1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1a24038814a45d9a4f233295b2494b8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/7a9975763a7042e4b2902d9001899be9] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.4 K 2024-11-24T02:48:26,969 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting faf96641872b4307bf6641b80d0de1c1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732416503474 2024-11-24T02:48:26,969 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a24038814a45d9a4f233295b2494b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732416503911 2024-11-24T02:48:26,970 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a9975763a7042e4b2902d9001899be9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732416504786 2024-11-24T02:48:26,978 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:26,978 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/6482a05ed86f4e4cb37cb15ec9472d9c is 50, key is test_row_0/A:col10/1732416506112/Put/seqid=0 2024-11-24T02:48:26,980 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:26,980 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/d937b7beb9234001a7e3e21f1e9254c7 is 50, key is test_row_0/B:col10/1732416506112/Put/seqid=0 2024-11-24T02:48:26,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742340_1516 (size=12949) 2024-11-24T02:48:26,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742341_1517 (size=12949) 2024-11-24T02:48:27,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:27,002 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:27,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:48:27,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/227fc29e31f84459b13378d6cc6415ea is 50, key is test_row_0/A:col10/1732416506168/Put/seqid=0 2024-11-24T02:48:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742342_1518 (size=14741) 2024-11-24T02:48:27,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:27,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:27,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T02:48:27,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416567057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416567058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416567176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416567176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416567336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416567336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416567336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:27,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T02:48:27,387 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/6482a05ed86f4e4cb37cb15ec9472d9c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/6482a05ed86f4e4cb37cb15ec9472d9c 2024-11-24T02:48:27,387 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/d937b7beb9234001a7e3e21f1e9254c7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/d937b7beb9234001a7e3e21f1e9254c7 2024-11-24T02:48:27,391 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into d937b7beb9234001a7e3e21f1e9254c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:27,391 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:27,391 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=13, startTime=1732416506966; duration=0sec 2024-11-24T02:48:27,391 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:27,391 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:27,391 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:27,392 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:27,392 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:27,392 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,392 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/4702861d11eb4bd6918ec81c74cb6c47, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/308b7ee89f8c472dbd0e3dea8bf9fa77, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/cbc8c2fd47da4b3fa44070b76da5be61] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.4 K 2024-11-24T02:48:27,392 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into 6482a05ed86f4e4cb37cb15ec9472d9c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:27,392 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:27,392 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=13, startTime=1732416506966; duration=0sec 2024-11-24T02:48:27,392 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:27,392 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:27,392 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4702861d11eb4bd6918ec81c74cb6c47, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732416503474 2024-11-24T02:48:27,393 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 308b7ee89f8c472dbd0e3dea8bf9fa77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732416503911 2024-11-24T02:48:27,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416567386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,393 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc8c2fd47da4b3fa44070b76da5be61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732416504786 2024-11-24T02:48:27,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416567386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,402 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:27,402 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/138875657bd1474a82cdd1e2723abdeb is 50, key is test_row_0/C:col10/1732416506112/Put/seqid=0 2024-11-24T02:48:27,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742343_1519 (size=12949) 2024-11-24T02:48:27,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/227fc29e31f84459b13378d6cc6415ea 2024-11-24T02:48:27,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/62065048d63a487f9083f30cc7754fc4 is 50, key is test_row_0/B:col10/1732416506168/Put/seqid=0 2024-11-24T02:48:27,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742344_1520 (size=12301) 2024-11-24T02:48:27,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:27,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:27,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416567697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416567697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:27,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:27,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:27,816 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/138875657bd1474a82cdd1e2723abdeb as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/138875657bd1474a82cdd1e2723abdeb 2024-11-24T02:48:27,819 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into 138875657bd1474a82cdd1e2723abdeb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:27,820 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:27,820 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=13, startTime=1732416506966; duration=0sec 2024-11-24T02:48:27,820 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:27,820 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:27,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/62065048d63a487f9083f30cc7754fc4 2024-11-24T02:48:27,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/2863e01dbe634992aeb170ca34c70503 is 50, key is test_row_0/C:col10/1732416506168/Put/seqid=0 2024-11-24T02:48:27,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742345_1521 (size=12301) 2024-11-24T02:48:27,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/2863e01dbe634992aeb170ca34c70503 2024-11-24T02:48:27,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/227fc29e31f84459b13378d6cc6415ea as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/227fc29e31f84459b13378d6cc6415ea 2024-11-24T02:48:27,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/227fc29e31f84459b13378d6cc6415ea, entries=200, sequenceid=305, filesize=14.4 K 2024-11-24T02:48:27,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/62065048d63a487f9083f30cc7754fc4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/62065048d63a487f9083f30cc7754fc4 2024-11-24T02:48:27,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/62065048d63a487f9083f30cc7754fc4, entries=150, sequenceid=305, filesize=12.0 K 2024-11-24T02:48:27,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/2863e01dbe634992aeb170ca34c70503 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/2863e01dbe634992aeb170ca34c70503 2024-11-24T02:48:27,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/2863e01dbe634992aeb170ca34c70503, entries=150, sequenceid=305, filesize=12.0 K 2024-11-24T02:48:27,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 5b9e19b406087483d14cf3e721c5b26c in 851ms, sequenceid=305, compaction requested=false 2024-11-24T02:48:27,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T02:48:27,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T02:48:27,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:27,962 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T02:48:27,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/938ca786d11b4b7e89e3a7cc260dfe33 is 50, key is test_row_0/A:col10/1732416507055/Put/seqid=0 2024-11-24T02:48:27,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742346_1522 (size=12301) 2024-11-24T02:48:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416568267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416568271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416568347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,352 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416568356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416568356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,362 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,363 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,371 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/938ca786d11b4b7e89e3a7cc260dfe33 2024-11-24T02:48:28,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/0959c44d84614b1cbd66a4241683cf06 is 50, key is test_row_0/B:col10/1732416507055/Put/seqid=0 2024-11-24T02:48:28,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742347_1523 (size=12301) 2024-11-24T02:48:28,382 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/0959c44d84614b1cbd66a4241683cf06 2024-11-24T02:48:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416568376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/040a8b5a625b4a1eab9d4c460253defc is 50, key is test_row_0/C:col10/1732416507055/Put/seqid=0 2024-11-24T02:48:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416568387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742348_1524 (size=12301) 2024-11-24T02:48:28,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416568597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416568597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,798 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/040a8b5a625b4a1eab9d4c460253defc 2024-11-24T02:48:28,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/938ca786d11b4b7e89e3a7cc260dfe33 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/938ca786d11b4b7e89e3a7cc260dfe33 2024-11-24T02:48:28,804 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/938ca786d11b4b7e89e3a7cc260dfe33, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T02:48:28,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/0959c44d84614b1cbd66a4241683cf06 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/0959c44d84614b1cbd66a4241683cf06 2024-11-24T02:48:28,807 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/0959c44d84614b1cbd66a4241683cf06, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T02:48:28,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/040a8b5a625b4a1eab9d4c460253defc as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/040a8b5a625b4a1eab9d4c460253defc 2024-11-24T02:48:28,810 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/040a8b5a625b4a1eab9d4c460253defc, entries=150, sequenceid=323, filesize=12.0 K 2024-11-24T02:48:28,811 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5b9e19b406087483d14cf3e721c5b26c in 849ms, sequenceid=323, compaction requested=true 2024-11-24T02:48:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:28,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-24T02:48:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-24T02:48:28,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-24T02:48:28,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-24T02:48:28,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.0760 sec 2024-11-24T02:48:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T02:48:28,886 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:28,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-24T02:48:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:48:28,888 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:28,889 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:28,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:28,910 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:28,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-24T02:48:28,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/fe98d5eae3a045b18d3e4ecf6cf2240b is 50, key is test_row_0/A:col10/1732416508260/Put/seqid=0 2024-11-24T02:48:28,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742349_1525 (size=14741) 2024-11-24T02:48:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416568947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416568951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:48:29,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416569067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:29,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416569067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:29,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:48:29,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416569276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,284 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:29,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416569287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:29,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/fe98d5eae3a045b18d3e4ecf6cf2240b 2024-11-24T02:48:29,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/267e845b59244715bd4bdab2a2ad05ec is 50, key is test_row_0/B:col10/1732416508260/Put/seqid=0 2024-11-24T02:48:29,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742350_1526 (size=12301) 2024-11-24T02:48:29,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:29,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:29,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:29,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:29,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:48:29,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416569597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416569597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:29,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/267e845b59244715bd4bdab2a2ad05ec 2024-11-24T02:48:29,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/5965edf3eb67423a910d6318e0addabe is 50, key is test_row_0/C:col10/1732416508260/Put/seqid=0 2024-11-24T02:48:29,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742351_1527 (size=12301) 2024-11-24T02:48:29,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:29,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:48:30,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416570107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416570107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:30,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:30,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:30,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:30,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:30,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:30,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:30,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/5965edf3eb67423a910d6318e0addabe 2024-11-24T02:48:30,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/fe98d5eae3a045b18d3e4ecf6cf2240b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/fe98d5eae3a045b18d3e4ecf6cf2240b 2024-11-24T02:48:30,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/fe98d5eae3a045b18d3e4ecf6cf2240b, entries=200, sequenceid=345, filesize=14.4 K 2024-11-24T02:48:30,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/267e845b59244715bd4bdab2a2ad05ec as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/267e845b59244715bd4bdab2a2ad05ec 2024-11-24T02:48:30,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/267e845b59244715bd4bdab2a2ad05ec, entries=150, sequenceid=345, filesize=12.0 K 2024-11-24T02:48:30,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/5965edf3eb67423a910d6318e0addabe as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/5965edf3eb67423a910d6318e0addabe 2024-11-24T02:48:30,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/5965edf3eb67423a910d6318e0addabe, entries=150, sequenceid=345, filesize=12.0 K 2024-11-24T02:48:30,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5b9e19b406087483d14cf3e721c5b26c in 1305ms, sequenceid=345, compaction requested=true 2024-11-24T02:48:30,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:30,216 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:30,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:30,216 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:30,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:30,217 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:30,217 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:30,217 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:30,217 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/6482a05ed86f4e4cb37cb15ec9472d9c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/227fc29e31f84459b13378d6cc6415ea, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/938ca786d11b4b7e89e3a7cc260dfe33, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/fe98d5eae3a045b18d3e4ecf6cf2240b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=53.4 K 2024-11-24T02:48:30,217 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6482a05ed86f4e4cb37cb15ec9472d9c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732416504786 2024-11-24T02:48:30,218 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 227fc29e31f84459b13378d6cc6415ea, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732416506153 2024-11-24T02:48:30,218 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:30,218 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:30,218 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:30,218 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/d937b7beb9234001a7e3e21f1e9254c7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/62065048d63a487f9083f30cc7754fc4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/0959c44d84614b1cbd66a4241683cf06, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/267e845b59244715bd4bdab2a2ad05ec] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=48.7 K 2024-11-24T02:48:30,218 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d937b7beb9234001a7e3e21f1e9254c7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732416504786 2024-11-24T02:48:30,218 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938ca786d11b4b7e89e3a7cc260dfe33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732416507044 2024-11-24T02:48:30,219 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 62065048d63a487f9083f30cc7754fc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732416506168 2024-11-24T02:48:30,219 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe98d5eae3a045b18d3e4ecf6cf2240b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732416508260 2024-11-24T02:48:30,219 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0959c44d84614b1cbd66a4241683cf06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732416507044 2024-11-24T02:48:30,219 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 267e845b59244715bd4bdab2a2ad05ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732416508260 2024-11-24T02:48:30,243 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:30,243 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/17b13c339fcd4eadbd9513a22806eef3 is 50, key is test_row_0/B:col10/1732416508260/Put/seqid=0 2024-11-24T02:48:30,247 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#450 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:30,247 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/906939384d9f47158bd136faf7393d17 is 50, key is test_row_0/A:col10/1732416508260/Put/seqid=0 2024-11-24T02:48:30,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742352_1528 (size=13085) 2024-11-24T02:48:30,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742353_1529 (size=13085) 2024-11-24T02:48:30,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-24T02:48:30,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:30,267 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:48:30,269 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/17b13c339fcd4eadbd9513a22806eef3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/17b13c339fcd4eadbd9513a22806eef3 2024-11-24T02:48:30,271 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/906939384d9f47158bd136faf7393d17 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/906939384d9f47158bd136faf7393d17 2024-11-24T02:48:30,279 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into 17b13c339fcd4eadbd9513a22806eef3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:30,279 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:30,279 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=12, startTime=1732416510216; duration=0sec 2024-11-24T02:48:30,279 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:30,279 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:30,279 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:30,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/7181f99ff3fb467e96598ce981b67ea0 is 50, key is test_row_0/A:col10/1732416508931/Put/seqid=0 2024-11-24T02:48:30,284 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into 906939384d9f47158bd136faf7393d17(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:30,284 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:30,284 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=12, startTime=1732416510215; duration=0sec 2024-11-24T02:48:30,285 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:30,285 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:30,285 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:30,285 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:30,285 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:30,285 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/138875657bd1474a82cdd1e2723abdeb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/2863e01dbe634992aeb170ca34c70503, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/040a8b5a625b4a1eab9d4c460253defc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/5965edf3eb67423a910d6318e0addabe] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=48.7 K 2024-11-24T02:48:30,285 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 138875657bd1474a82cdd1e2723abdeb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1732416504786 2024-11-24T02:48:30,286 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2863e01dbe634992aeb170ca34c70503, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732416506168 2024-11-24T02:48:30,286 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 040a8b5a625b4a1eab9d4c460253defc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732416507044 2024-11-24T02:48:30,287 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5965edf3eb67423a910d6318e0addabe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732416508260 2024-11-24T02:48:30,307 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:30,307 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/c9267899eaee43c78ecf1ba9f506f131 is 50, key is test_row_0/C:col10/1732416508260/Put/seqid=0 2024-11-24T02:48:30,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742354_1530 (size=12301) 2024-11-24T02:48:30,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742355_1531 (size=13085) 2024-11-24T02:48:30,333 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/c9267899eaee43c78ecf1ba9f506f131 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/c9267899eaee43c78ecf1ba9f506f131 2024-11-24T02:48:30,339 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into c9267899eaee43c78ecf1ba9f506f131(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:30,339 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:30,339 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=12, startTime=1732416510217; duration=0sec 2024-11-24T02:48:30,339 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:30,339 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:30,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416570430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416570432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416570432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416570546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416570546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416570546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,710 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/7181f99ff3fb467e96598ce981b67ea0 2024-11-24T02:48:30,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/a621add2b12540a0a8a6d4542f12170a is 50, key is test_row_0/B:col10/1732416508931/Put/seqid=0 2024-11-24T02:48:30,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742356_1532 (size=12301) 2024-11-24T02:48:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416570756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416570756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416570756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:48:31,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416571068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416571076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416571076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416571117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416571117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,121 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:31,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,121 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,131 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/a621add2b12540a0a8a6d4542f12170a 2024-11-24T02:48:31,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/356c31deb5e147189441a4dee55c4658 is 50, key is test_row_0/C:col10/1732416508931/Put/seqid=0 2024-11-24T02:48:31,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742357_1533 (size=12301) 2024-11-24T02:48:31,169 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/356c31deb5e147189441a4dee55c4658 2024-11-24T02:48:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/7181f99ff3fb467e96598ce981b67ea0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/7181f99ff3fb467e96598ce981b67ea0 2024-11-24T02:48:31,179 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/7181f99ff3fb467e96598ce981b67ea0, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:48:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/a621add2b12540a0a8a6d4542f12170a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a621add2b12540a0a8a6d4542f12170a 2024-11-24T02:48:31,189 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a621add2b12540a0a8a6d4542f12170a, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:48:31,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/356c31deb5e147189441a4dee55c4658 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/356c31deb5e147189441a4dee55c4658 2024-11-24T02:48:31,204 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/356c31deb5e147189441a4dee55c4658, entries=150, sequenceid=359, filesize=12.0 K 2024-11-24T02:48:31,205 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5b9e19b406087483d14cf3e721c5b26c in 938ms, sequenceid=359, compaction requested=false 2024-11-24T02:48:31,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:31,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:31,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-24T02:48:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-24T02:48:31,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-24T02:48:31,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-24T02:48:31,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 2.3250 sec 2024-11-24T02:48:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:31,593 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:31,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-24T02:48:31,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/4043936099c040c19f86c1639807c48b is 50, key is test_row_0/A:col10/1732416510428/Put/seqid=0 2024-11-24T02:48:31,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742358_1534 (size=14741) 2024-11-24T02:48:31,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/4043936099c040c19f86c1639807c48b 2024-11-24T02:48:31,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/005f1a5f78b043e78d9542a10f1194e6 is 50, key is test_row_0/B:col10/1732416510428/Put/seqid=0 2024-11-24T02:48:31,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416571630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416571631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416571637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742359_1535 (size=12301) 2024-11-24T02:48:31,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/005f1a5f78b043e78d9542a10f1194e6 2024-11-24T02:48:31,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/7f72baf05bda40c4951662e1248aa0da is 50, key is test_row_0/C:col10/1732416510428/Put/seqid=0 2024-11-24T02:48:31,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742360_1536 (size=12301) 2024-11-24T02:48:31,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/7f72baf05bda40c4951662e1248aa0da 2024-11-24T02:48:31,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/4043936099c040c19f86c1639807c48b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4043936099c040c19f86c1639807c48b 2024-11-24T02:48:31,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4043936099c040c19f86c1639807c48b, entries=200, sequenceid=385, filesize=14.4 K 2024-11-24T02:48:31,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/005f1a5f78b043e78d9542a10f1194e6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/005f1a5f78b043e78d9542a10f1194e6 2024-11-24T02:48:31,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/005f1a5f78b043e78d9542a10f1194e6, entries=150, sequenceid=385, filesize=12.0 K 2024-11-24T02:48:31,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/7f72baf05bda40c4951662e1248aa0da as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/7f72baf05bda40c4951662e1248aa0da 2024-11-24T02:48:31,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/7f72baf05bda40c4951662e1248aa0da, entries=150, sequenceid=385, filesize=12.0 K 2024-11-24T02:48:31,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5b9e19b406087483d14cf3e721c5b26c in 142ms, sequenceid=385, compaction requested=true 2024-11-24T02:48:31,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:31,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:31,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:31,736 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:31,736 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:31,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:31,745 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:31,745 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:31,745 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:31,745 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:31,745 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:31,745 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:31,745 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/906939384d9f47158bd136faf7393d17, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/7181f99ff3fb467e96598ce981b67ea0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4043936099c040c19f86c1639807c48b] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=39.2 K 2024-11-24T02:48:31,745 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/17b13c339fcd4eadbd9513a22806eef3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a621add2b12540a0a8a6d4542f12170a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/005f1a5f78b043e78d9542a10f1194e6] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.8 K 2024-11-24T02:48:31,745 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 17b13c339fcd4eadbd9513a22806eef3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732416508260 2024-11-24T02:48:31,746 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 906939384d9f47158bd136faf7393d17, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732416508260 2024-11-24T02:48:31,746 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting a621add2b12540a0a8a6d4542f12170a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416508931 2024-11-24T02:48:31,746 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7181f99ff3fb467e96598ce981b67ea0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416508931 2024-11-24T02:48:31,746 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 005f1a5f78b043e78d9542a10f1194e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732416510428 2024-11-24T02:48:31,746 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4043936099c040c19f86c1639807c48b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732416510428 2024-11-24T02:48:31,753 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:31,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-24T02:48:31,757 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:31,757 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/d4a915ccdc6b4805841ad440273601ce is 50, key is test_row_0/A:col10/1732416510428/Put/seqid=0 2024-11-24T02:48:31,760 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:31,760 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/edf3329cda1943758aef1ef2ff36499f is 50, key is test_row_0/B:col10/1732416510428/Put/seqid=0 2024-11-24T02:48:31,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f44045fa0ac54097a353a56585721d21 is 50, key is test_row_0/A:col10/1732416511751/Put/seqid=0 2024-11-24T02:48:31,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742361_1537 (size=13187) 2024-11-24T02:48:31,814 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/d4a915ccdc6b4805841ad440273601ce as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/d4a915ccdc6b4805841ad440273601ce 2024-11-24T02:48:31,821 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into d4a915ccdc6b4805841ad440273601ce(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:31,821 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:31,821 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=13, startTime=1732416511736; duration=0sec 2024-11-24T02:48:31,822 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:31,822 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:31,822 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:31,823 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:31,823 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:31,823 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:31,823 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/c9267899eaee43c78ecf1ba9f506f131, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/356c31deb5e147189441a4dee55c4658, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/7f72baf05bda40c4951662e1248aa0da] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.8 K 2024-11-24T02:48:31,823 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9267899eaee43c78ecf1ba9f506f131, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732416508260 2024-11-24T02:48:31,823 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356c31deb5e147189441a4dee55c4658, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1732416508931 2024-11-24T02:48:31,824 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f72baf05bda40c4951662e1248aa0da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732416510428 2024-11-24T02:48:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742362_1538 (size=13187) 2024-11-24T02:48:31,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742363_1539 (size=14741) 2024-11-24T02:48:31,855 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:31,855 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a419759ce2e64e8c812331c7d5a98a99 is 50, key is test_row_0/C:col10/1732416510428/Put/seqid=0 2024-11-24T02:48:31,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416571847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742364_1540 (size=13187) 2024-11-24T02:48:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416571866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416571868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416571976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416571987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416571987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:32,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416572197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:32,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416572207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416572207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:32,232 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/edf3329cda1943758aef1ef2ff36499f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/edf3329cda1943758aef1ef2ff36499f 2024-11-24T02:48:32,237 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into edf3329cda1943758aef1ef2ff36499f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:32,237 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:32,237 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=13, startTime=1732416511736; duration=0sec 2024-11-24T02:48:32,237 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:32,237 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:32,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f44045fa0ac54097a353a56585721d21 2024-11-24T02:48:32,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/57cfec8dcbf848a1b63fb8ae24ea6f52 is 50, key is test_row_0/B:col10/1732416511751/Put/seqid=0 2024-11-24T02:48:32,278 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a419759ce2e64e8c812331c7d5a98a99 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a419759ce2e64e8c812331c7d5a98a99 2024-11-24T02:48:32,283 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into a419759ce2e64e8c812331c7d5a98a99(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:32,283 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:32,283 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=13, startTime=1732416511737; duration=0sec 2024-11-24T02:48:32,283 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:32,283 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:32,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742365_1541 (size=12301) 2024-11-24T02:48:32,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416572516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:32,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416572516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:32,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416572517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:32,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:32,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/57cfec8dcbf848a1b63fb8ae24ea6f52 2024-11-24T02:48:32,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a692db1471ef49059dfa489c1093cb9f is 50, key is test_row_0/C:col10/1732416511751/Put/seqid=0 2024-11-24T02:48:32,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742366_1542 (size=12301) 2024-11-24T02:48:32,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a692db1471ef49059dfa489c1093cb9f 2024-11-24T02:48:32,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/f44045fa0ac54097a353a56585721d21 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f44045fa0ac54097a353a56585721d21 2024-11-24T02:48:32,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f44045fa0ac54097a353a56585721d21, entries=200, sequenceid=399, filesize=14.4 K 2024-11-24T02:48:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/57cfec8dcbf848a1b63fb8ae24ea6f52 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/57cfec8dcbf848a1b63fb8ae24ea6f52 2024-11-24T02:48:32,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/57cfec8dcbf848a1b63fb8ae24ea6f52, entries=150, sequenceid=399, filesize=12.0 K 2024-11-24T02:48:32,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/a692db1471ef49059dfa489c1093cb9f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a692db1471ef49059dfa489c1093cb9f 2024-11-24T02:48:32,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a692db1471ef49059dfa489c1093cb9f, entries=150, sequenceid=399, filesize=12.0 K 2024-11-24T02:48:32,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5b9e19b406087483d14cf3e721c5b26c in 998ms, sequenceid=399, compaction requested=false 2024-11-24T02:48:32,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T02:48:33,057 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:33,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-24T02:48:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T02:48:33,060 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:33,061 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:33,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:33,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:48:33,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/c1966b8abd58407282623246c22f8213 is 50, key is test_row_0/A:col10/1732416511840/Put/seqid=0 2024-11-24T02:48:33,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742367_1543 (size=12301) 2024-11-24T02:48:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416573109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416573114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416573111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34244 deadline: 1732416573129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,139 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732416573147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,152 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 5b9e19b406087483d14cf3e721c5b26c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T02:48:33,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416573236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416573236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416573236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T02:48:33,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416573446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416573446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416573456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/c1966b8abd58407282623246c22f8213 2024-11-24T02:48:33,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/1df1d2262d744d0f926b493661b71bf0 is 50, key is test_row_0/B:col10/1732416511840/Put/seqid=0 2024-11-24T02:48:33,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742368_1544 (size=12301) 2024-11-24T02:48:33,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:33,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T02:48:33,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416573756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416573767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416573777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:33,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/1df1d2262d744d0f926b493661b71bf0 2024-11-24T02:48:33,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/12c7e00a727f4829be3781c42b9fac04 is 50, key is test_row_0/C:col10/1732416511840/Put/seqid=0 2024-11-24T02:48:33,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742369_1545 (size=12301) 2024-11-24T02:48:33,967 INFO [Thread-2065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2065. 2024-11-24T02:48:33,967 DEBUG [Thread-2065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:33,967 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:33,967 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,967 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-24T02:48:33,968 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:33,968 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,968 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:33,972 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-24T02:48:33,972 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:33,972 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,972 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:33,975 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-24T02:48:33,975 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:33,975 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,975 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:33,976 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-11-24T02:48:33,976 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:33,976 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,976 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:33,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:33,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:33,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:34,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:34,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T02:48:34,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34326 deadline: 1732416574267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:34,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34254 deadline: 1732416574287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:34,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:34,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:34,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1732416574297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 , the old value is region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=2 because the exception is null or not the one we care about 2024-11-24T02:48:34,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/12c7e00a727f4829be3781c42b9fac04 2024-11-24T02:48:34,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/c1966b8abd58407282623246c22f8213 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c1966b8abd58407282623246c22f8213 2024-11-24T02:48:34,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c1966b8abd58407282623246c22f8213, entries=150, sequenceid=427, filesize=12.0 K 2024-11-24T02:48:34,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/1df1d2262d744d0f926b493661b71bf0 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1df1d2262d744d0f926b493661b71bf0 2024-11-24T02:48:34,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1df1d2262d744d0f926b493661b71bf0, entries=150, sequenceid=427, filesize=12.0 K 2024-11-24T02:48:34,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/12c7e00a727f4829be3781c42b9fac04 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/12c7e00a727f4829be3781c42b9fac04 2024-11-24T02:48:34,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/12c7e00a727f4829be3781c42b9fac04, entries=150, sequenceid=427, filesize=12.0 K 2024-11-24T02:48:34,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5b9e19b406087483d14cf3e721c5b26c in 1255ms, sequenceid=427, compaction requested=true 2024-11-24T02:48:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:34,331 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b9e19b406087483d14cf3e721c5b26c:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:34,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:34,331 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:34,332 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:34,332 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:34,332 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/A is initiating minor compaction (all files) 2024-11-24T02:48:34,332 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/A in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,332 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/B is initiating minor compaction (all files) 2024-11-24T02:48:34,332 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/d4a915ccdc6b4805841ad440273601ce, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f44045fa0ac54097a353a56585721d21, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c1966b8abd58407282623246c22f8213] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=39.3 K 2024-11-24T02:48:34,332 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/B in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,332 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/edf3329cda1943758aef1ef2ff36499f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/57cfec8dcbf848a1b63fb8ae24ea6f52, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1df1d2262d744d0f926b493661b71bf0] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.9 K 2024-11-24T02:48:34,332 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4a915ccdc6b4805841ad440273601ce, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732416510428 2024-11-24T02:48:34,333 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting edf3329cda1943758aef1ef2ff36499f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732416510428 2024-11-24T02:48:34,333 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44045fa0ac54097a353a56585721d21, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732416511599 2024-11-24T02:48:34,333 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 57cfec8dcbf848a1b63fb8ae24ea6f52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732416511627 2024-11-24T02:48:34,333 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1966b8abd58407282623246c22f8213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732416511840 2024-11-24T02:48:34,333 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1df1d2262d744d0f926b493661b71bf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732416511840 2024-11-24T02:48:34,340 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:34,340 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#A#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:34,340 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/bef28d84e1ae475fa43c625a541433e7 is 50, key is test_row_0/B:col10/1732416511840/Put/seqid=0 2024-11-24T02:48:34,340 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/52a51290415e4c67924a4cefcca36806 is 50, key is test_row_0/A:col10/1732416511840/Put/seqid=0 2024-11-24T02:48:34,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742371_1547 (size=13289) 2024-11-24T02:48:34,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742370_1546 (size=13289) 2024-11-24T02:48:34,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-24T02:48:34,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,444 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:48:34,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/028460835be2418799a293f2d351f68e is 50, key is test_row_0/A:col10/1732416513108/Put/seqid=0 2024-11-24T02:48:34,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742372_1548 (size=12301) 2024-11-24T02:48:34,747 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/bef28d84e1ae475fa43c625a541433e7 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/bef28d84e1ae475fa43c625a541433e7 2024-11-24T02:48:34,747 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/52a51290415e4c67924a4cefcca36806 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/52a51290415e4c67924a4cefcca36806 2024-11-24T02:48:34,750 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/A of 5b9e19b406087483d14cf3e721c5b26c into 52a51290415e4c67924a4cefcca36806(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:34,750 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/B of 5b9e19b406087483d14cf3e721c5b26c into bef28d84e1ae475fa43c625a541433e7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:34,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:34,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:34,750 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/A, priority=13, startTime=1732416514331; duration=0sec 2024-11-24T02:48:34,750 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/B, priority=13, startTime=1732416514331; duration=0sec 2024-11-24T02:48:34,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:34,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:A 2024-11-24T02:48:34,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:34,750 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:B 2024-11-24T02:48:34,750 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:34,751 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:34,751 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 5b9e19b406087483d14cf3e721c5b26c/C is initiating minor compaction (all files) 2024-11-24T02:48:34,751 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b9e19b406087483d14cf3e721c5b26c/C in TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:34,751 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a419759ce2e64e8c812331c7d5a98a99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a692db1471ef49059dfa489c1093cb9f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/12c7e00a727f4829be3781c42b9fac04] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp, totalSize=36.9 K 2024-11-24T02:48:34,751 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a419759ce2e64e8c812331c7d5a98a99, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732416510428 2024-11-24T02:48:34,752 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a692db1471ef49059dfa489c1093cb9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732416511627 2024-11-24T02:48:34,752 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12c7e00a727f4829be3781c42b9fac04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732416511840 2024-11-24T02:48:34,758 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b9e19b406087483d14cf3e721c5b26c#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:34,759 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/91f49fdfced44f73a086634fa17e47c3 is 50, key is test_row_0/C:col10/1732416511840/Put/seqid=0 2024-11-24T02:48:34,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742373_1549 (size=13289) 2024-11-24T02:48:34,857 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/028460835be2418799a293f2d351f68e 2024-11-24T02:48:34,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c517018b78e6478482f72910cda73d79 is 50, key is test_row_0/B:col10/1732416513108/Put/seqid=0 2024-11-24T02:48:34,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742374_1550 (size=12301) 2024-11-24T02:48:35,169 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/91f49fdfced44f73a086634fa17e47c3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/91f49fdfced44f73a086634fa17e47c3 2024-11-24T02:48:35,178 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b9e19b406087483d14cf3e721c5b26c/C of 5b9e19b406087483d14cf3e721c5b26c into 91f49fdfced44f73a086634fa17e47c3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:35,178 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:35,178 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., storeName=5b9e19b406087483d14cf3e721c5b26c/C, priority=13, startTime=1732416514331; duration=0sec 2024-11-24T02:48:35,178 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:35,178 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b9e19b406087483d14cf3e721c5b26c:C 2024-11-24T02:48:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T02:48:35,275 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c517018b78e6478482f72910cda73d79 2024-11-24T02:48:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:35,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. as already flushing 2024-11-24T02:48:35,282 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-11-24T02:48:35,282 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:35,282 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:35,282 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:35,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/0682499bfb2246edbd9fcd68ffb76a4a is 50, key is test_row_0/C:col10/1732416513108/Put/seqid=0 2024-11-24T02:48:35,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742375_1551 (size=12301) 2024-11-24T02:48:35,308 INFO [Thread-2058 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2058. 2024-11-24T02:48:35,308 DEBUG [Thread-2058 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:35,308 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:35,308 DEBUG [Thread-2058 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:35,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:35,319 INFO [Thread-2054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2054. 2024-11-24T02:48:35,319 DEBUG [Thread-2054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:35,319 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:35,319 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:35,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:35,691 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/0682499bfb2246edbd9fcd68ffb76a4a 2024-11-24T02:48:35,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/028460835be2418799a293f2d351f68e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/028460835be2418799a293f2d351f68e 2024-11-24T02:48:35,705 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/028460835be2418799a293f2d351f68e, entries=150, sequenceid=438, filesize=12.0 K 2024-11-24T02:48:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/c517018b78e6478482f72910cda73d79 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c517018b78e6478482f72910cda73d79 2024-11-24T02:48:35,708 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c517018b78e6478482f72910cda73d79, entries=150, sequenceid=438, filesize=12.0 K 2024-11-24T02:48:35,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/0682499bfb2246edbd9fcd68ffb76a4a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0682499bfb2246edbd9fcd68ffb76a4a 2024-11-24T02:48:35,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0682499bfb2246edbd9fcd68ffb76a4a, entries=150, sequenceid=438, filesize=12.0 K 2024-11-24T02:48:35,711 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=20.13 KB/20610 for 5b9e19b406087483d14cf3e721c5b26c in 1267ms, sequenceid=438, compaction requested=false 2024-11-24T02:48:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 5b9e19b406087483d14cf3e721c5b26c: 2024-11-24T02:48:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-24T02:48:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-24T02:48:35,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-24T02:48:35,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6520 sec 2024-11-24T02:48:35,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 2.6560 sec 2024-11-24T02:48:37,159 INFO [Thread-2056 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2056. 2024-11-24T02:48:37,159 DEBUG [Thread-2056 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:37,159 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:37,159 DEBUG [Thread-2056 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:37,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:37,179 INFO [Thread-2052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2052. 2024-11-24T02:48:37,179 DEBUG [Thread-2052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:48:37,179 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:37,179 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:37,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-24T02:48:37,228 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1174 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3522 rows 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3502 rows 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1164 2024-11-24T02:48:37,228 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3492 rows 2024-11-24T02:48:37,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1171 2024-11-24T02:48:37,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3513 rows 2024-11-24T02:48:37,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1175 2024-11-24T02:48:37,229 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3520 rows 2024-11-24T02:48:37,229 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:48:37,229 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:48:37,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:37,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:37,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:48:37,231 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:48:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T02:48:37,237 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416517237"}]},"ts":"1732416517237"} 2024-11-24T02:48:37,240 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:48:37,240 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:48:37,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:48:37,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, UNASSIGN}] 2024-11-24T02:48:37,244 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, UNASSIGN 2024-11-24T02:48:37,245 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5b9e19b406087483d14cf3e721c5b26c, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:37,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, UNASSIGN because future has completed 2024-11-24T02:48:37,247 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:48:37,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:48:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T02:48:37,400 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:37,400 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:48:37,400 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 5b9e19b406087483d14cf3e721c5b26c, disabling compactions & flushes 2024-11-24T02:48:37,400 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:37,400 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:37,400 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. after waiting 0 ms 2024-11-24T02:48:37,400 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:37,401 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(2902): Flushing 5b9e19b406087483d14cf3e721c5b26c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-24T02:48:37,410 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/1947fcb923ed4d67b4329d2db322d8e9 is 50, key is test_row_0/A:col10/1732416515307/Put/seqid=0 2024-11-24T02:48:37,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742376_1552 (size=12301) 2024-11-24T02:48:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T02:48:37,815 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/1947fcb923ed4d67b4329d2db322d8e9 2024-11-24T02:48:37,828 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/b9de4483b866435b915d7f335709f59d is 50, key is test_row_0/B:col10/1732416515307/Put/seqid=0 2024-11-24T02:48:37,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742377_1553 (size=12301) 2024-11-24T02:48:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T02:48:38,106 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:48:38,232 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/b9de4483b866435b915d7f335709f59d 2024-11-24T02:48:38,238 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/de823a8a6a2a4f368f99cb799ef1a2ef is 50, key is test_row_0/C:col10/1732416515307/Put/seqid=0 2024-11-24T02:48:38,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742378_1554 (size=12301) 2024-11-24T02:48:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T02:48:38,641 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/de823a8a6a2a4f368f99cb799ef1a2ef 2024-11-24T02:48:38,645 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/A/1947fcb923ed4d67b4329d2db322d8e9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/1947fcb923ed4d67b4329d2db322d8e9 2024-11-24T02:48:38,648 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/1947fcb923ed4d67b4329d2db322d8e9, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T02:48:38,648 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/B/b9de4483b866435b915d7f335709f59d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/b9de4483b866435b915d7f335709f59d 2024-11-24T02:48:38,651 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/b9de4483b866435b915d7f335709f59d, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T02:48:38,652 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/.tmp/C/de823a8a6a2a4f368f99cb799ef1a2ef as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/de823a8a6a2a4f368f99cb799ef1a2ef 2024-11-24T02:48:38,654 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/de823a8a6a2a4f368f99cb799ef1a2ef, entries=150, sequenceid=449, filesize=12.0 K 2024-11-24T02:48:38,655 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5b9e19b406087483d14cf3e721c5b26c in 1255ms, sequenceid=449, compaction requested=true 2024-11-24T02:48:38,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/8e5d7e33f4e84005bd321a4b226fcd6f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/3f122b0530cb47e0b1147f41e3440c3e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c20c9f0f223e49598a013c674938e4ea, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/22ab3b51e67c41b2b83e95ea76b7f093, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/36e38a9d9f884a82a38b7c8f214b32cf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/b1b59ca9da58492e83f6acff6f624032, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/590fd4db60fc4136ae75245ed0c8b082, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4610286483ae4516ab53735530c63575, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2b3d97cb5a544308adfd6415ec93d152, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/a1dc2949116c4c9f920e4d38a61dd12a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/641022970b62450b9ca753c6ad745d93, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2d33521384614614bf0bf197fb0a52e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/9d405acf2161459a98bbd4039cb46aa2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/99d305c7b8404436b053afb310d889cb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/eb480f627eec4039a4ee65a19c4da7dc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f184dd6548ec4335b77f40aaf203414e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/ac53f8f794f6437e84cc046e645d6ff8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f62de0daf3c646a0afed0f4c5da6c19e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/47cbc1536c3240f19f7b6836c717142b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/6482a05ed86f4e4cb37cb15ec9472d9c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/227fc29e31f84459b13378d6cc6415ea, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/938ca786d11b4b7e89e3a7cc260dfe33, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/fe98d5eae3a045b18d3e4ecf6cf2240b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/906939384d9f47158bd136faf7393d17, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/7181f99ff3fb467e96598ce981b67ea0, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4043936099c040c19f86c1639807c48b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/d4a915ccdc6b4805841ad440273601ce, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f44045fa0ac54097a353a56585721d21, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c1966b8abd58407282623246c22f8213] to archive 2024-11-24T02:48:38,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:48:38,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/8e5d7e33f4e84005bd321a4b226fcd6f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/8e5d7e33f4e84005bd321a4b226fcd6f 2024-11-24T02:48:38,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/3f122b0530cb47e0b1147f41e3440c3e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/3f122b0530cb47e0b1147f41e3440c3e 2024-11-24T02:48:38,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c20c9f0f223e49598a013c674938e4ea to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c20c9f0f223e49598a013c674938e4ea 2024-11-24T02:48:38,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/22ab3b51e67c41b2b83e95ea76b7f093 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/22ab3b51e67c41b2b83e95ea76b7f093 2024-11-24T02:48:38,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/36e38a9d9f884a82a38b7c8f214b32cf to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/36e38a9d9f884a82a38b7c8f214b32cf 2024-11-24T02:48:38,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/b1b59ca9da58492e83f6acff6f624032 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/b1b59ca9da58492e83f6acff6f624032 2024-11-24T02:48:38,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/590fd4db60fc4136ae75245ed0c8b082 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/590fd4db60fc4136ae75245ed0c8b082 2024-11-24T02:48:38,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4610286483ae4516ab53735530c63575 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4610286483ae4516ab53735530c63575 2024-11-24T02:48:38,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2b3d97cb5a544308adfd6415ec93d152 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2b3d97cb5a544308adfd6415ec93d152 2024-11-24T02:48:38,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/a1dc2949116c4c9f920e4d38a61dd12a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/a1dc2949116c4c9f920e4d38a61dd12a 2024-11-24T02:48:38,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/641022970b62450b9ca753c6ad745d93 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/641022970b62450b9ca753c6ad745d93 2024-11-24T02:48:38,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2d33521384614614bf0bf197fb0a52e6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/2d33521384614614bf0bf197fb0a52e6 2024-11-24T02:48:38,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/9d405acf2161459a98bbd4039cb46aa2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/9d405acf2161459a98bbd4039cb46aa2 2024-11-24T02:48:38,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/99d305c7b8404436b053afb310d889cb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/99d305c7b8404436b053afb310d889cb 2024-11-24T02:48:38,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/eb480f627eec4039a4ee65a19c4da7dc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/eb480f627eec4039a4ee65a19c4da7dc 2024-11-24T02:48:38,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f184dd6548ec4335b77f40aaf203414e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f184dd6548ec4335b77f40aaf203414e 2024-11-24T02:48:38,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/ac53f8f794f6437e84cc046e645d6ff8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/ac53f8f794f6437e84cc046e645d6ff8 2024-11-24T02:48:38,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f62de0daf3c646a0afed0f4c5da6c19e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f62de0daf3c646a0afed0f4c5da6c19e 2024-11-24T02:48:38,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/47cbc1536c3240f19f7b6836c717142b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/47cbc1536c3240f19f7b6836c717142b 2024-11-24T02:48:38,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/6482a05ed86f4e4cb37cb15ec9472d9c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/6482a05ed86f4e4cb37cb15ec9472d9c 2024-11-24T02:48:38,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/227fc29e31f84459b13378d6cc6415ea to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/227fc29e31f84459b13378d6cc6415ea 2024-11-24T02:48:38,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/938ca786d11b4b7e89e3a7cc260dfe33 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/938ca786d11b4b7e89e3a7cc260dfe33 2024-11-24T02:48:38,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/fe98d5eae3a045b18d3e4ecf6cf2240b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/fe98d5eae3a045b18d3e4ecf6cf2240b 2024-11-24T02:48:38,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/906939384d9f47158bd136faf7393d17 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/906939384d9f47158bd136faf7393d17 2024-11-24T02:48:38,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/7181f99ff3fb467e96598ce981b67ea0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/7181f99ff3fb467e96598ce981b67ea0 2024-11-24T02:48:38,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4043936099c040c19f86c1639807c48b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/4043936099c040c19f86c1639807c48b 2024-11-24T02:48:38,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/d4a915ccdc6b4805841ad440273601ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/d4a915ccdc6b4805841ad440273601ce 2024-11-24T02:48:38,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f44045fa0ac54097a353a56585721d21 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/f44045fa0ac54097a353a56585721d21 2024-11-24T02:48:38,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c1966b8abd58407282623246c22f8213 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/c1966b8abd58407282623246c22f8213 2024-11-24T02:48:38,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f17d8e7cb3e14798ac31cded668c8096, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6ba069fdd214144b0d16ab57bada65e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1dfe70fa06a49ccbda33d12710eda05, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/8d6fde395f5043058affa597842bd263, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/2ce8c8949c9c4e35a982fb1561869639, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/fc3130de10c846c9a5a9e11518c7e31e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/74411638272d4ac4ba9def90859f72b5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/6a8dc8418ea84f909952a0f671609e5c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/4860fec3fe1b4d33b646b952a53425fa, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1ecba8ba26f6472d9e151ac8aad85930, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a82962771b2b48f8b79d3c5b9f231eff, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f76293c8bd59494eafc175c80217f1dd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6da7da3987941688fbe29f0dc515e1b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/df291f72a492431c9d37193c81bfd29d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e1b89a5b9fa34b27ad60d7b0c448dd99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/faf96641872b4307bf6641b80d0de1c1, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e490404198a949f18b382c05b3b9f6ba, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1a24038814a45d9a4f233295b2494b8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/d937b7beb9234001a7e3e21f1e9254c7, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/7a9975763a7042e4b2902d9001899be9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/62065048d63a487f9083f30cc7754fc4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/0959c44d84614b1cbd66a4241683cf06, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/17b13c339fcd4eadbd9513a22806eef3, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/267e845b59244715bd4bdab2a2ad05ec, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a621add2b12540a0a8a6d4542f12170a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/edf3329cda1943758aef1ef2ff36499f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/005f1a5f78b043e78d9542a10f1194e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/57cfec8dcbf848a1b63fb8ae24ea6f52, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1df1d2262d744d0f926b493661b71bf0] to archive 2024-11-24T02:48:38,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:48:38,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f17d8e7cb3e14798ac31cded668c8096 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f17d8e7cb3e14798ac31cded668c8096 2024-11-24T02:48:38,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6ba069fdd214144b0d16ab57bada65e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6ba069fdd214144b0d16ab57bada65e 2024-11-24T02:48:38,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1dfe70fa06a49ccbda33d12710eda05 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1dfe70fa06a49ccbda33d12710eda05 2024-11-24T02:48:38,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/8d6fde395f5043058affa597842bd263 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/8d6fde395f5043058affa597842bd263 2024-11-24T02:48:38,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/2ce8c8949c9c4e35a982fb1561869639 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/2ce8c8949c9c4e35a982fb1561869639 2024-11-24T02:48:38,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/fc3130de10c846c9a5a9e11518c7e31e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/fc3130de10c846c9a5a9e11518c7e31e 2024-11-24T02:48:38,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/74411638272d4ac4ba9def90859f72b5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/74411638272d4ac4ba9def90859f72b5 2024-11-24T02:48:38,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/6a8dc8418ea84f909952a0f671609e5c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/6a8dc8418ea84f909952a0f671609e5c 2024-11-24T02:48:38,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/4860fec3fe1b4d33b646b952a53425fa to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/4860fec3fe1b4d33b646b952a53425fa 2024-11-24T02:48:38,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1ecba8ba26f6472d9e151ac8aad85930 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1ecba8ba26f6472d9e151ac8aad85930 2024-11-24T02:48:38,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a82962771b2b48f8b79d3c5b9f231eff to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a82962771b2b48f8b79d3c5b9f231eff 2024-11-24T02:48:38,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f76293c8bd59494eafc175c80217f1dd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/f76293c8bd59494eafc175c80217f1dd 2024-11-24T02:48:38,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6da7da3987941688fbe29f0dc515e1b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e6da7da3987941688fbe29f0dc515e1b 2024-11-24T02:48:38,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/df291f72a492431c9d37193c81bfd29d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/df291f72a492431c9d37193c81bfd29d 2024-11-24T02:48:38,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e1b89a5b9fa34b27ad60d7b0c448dd99 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e1b89a5b9fa34b27ad60d7b0c448dd99 2024-11-24T02:48:38,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/faf96641872b4307bf6641b80d0de1c1 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/faf96641872b4307bf6641b80d0de1c1 2024-11-24T02:48:38,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e490404198a949f18b382c05b3b9f6ba to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/e490404198a949f18b382c05b3b9f6ba 2024-11-24T02:48:38,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1a24038814a45d9a4f233295b2494b8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c1a24038814a45d9a4f233295b2494b8 2024-11-24T02:48:38,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/d937b7beb9234001a7e3e21f1e9254c7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/d937b7beb9234001a7e3e21f1e9254c7 2024-11-24T02:48:38,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/7a9975763a7042e4b2902d9001899be9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/7a9975763a7042e4b2902d9001899be9 2024-11-24T02:48:38,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/62065048d63a487f9083f30cc7754fc4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/62065048d63a487f9083f30cc7754fc4 2024-11-24T02:48:38,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/0959c44d84614b1cbd66a4241683cf06 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/0959c44d84614b1cbd66a4241683cf06 2024-11-24T02:48:38,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/17b13c339fcd4eadbd9513a22806eef3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/17b13c339fcd4eadbd9513a22806eef3 2024-11-24T02:48:38,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/267e845b59244715bd4bdab2a2ad05ec to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/267e845b59244715bd4bdab2a2ad05ec 2024-11-24T02:48:38,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a621add2b12540a0a8a6d4542f12170a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/a621add2b12540a0a8a6d4542f12170a 2024-11-24T02:48:38,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/edf3329cda1943758aef1ef2ff36499f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/edf3329cda1943758aef1ef2ff36499f 2024-11-24T02:48:38,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/005f1a5f78b043e78d9542a10f1194e6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/005f1a5f78b043e78d9542a10f1194e6 2024-11-24T02:48:38,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/57cfec8dcbf848a1b63fb8ae24ea6f52 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/57cfec8dcbf848a1b63fb8ae24ea6f52 2024-11-24T02:48:38,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1df1d2262d744d0f926b493661b71bf0 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/1df1d2262d744d0f926b493661b71bf0 2024-11-24T02:48:38,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0d528b8a31674907bc4eb92704f0b83a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/27e63cb180a04225ba490ad04ab1bffd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8c8ccf6c5fcb410a9e9f414398d6084c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/ae9772cc41c449e68c3618da07128046, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/1f500698d95b4bedb843792882939648, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a8004b6d59a1460b8f983fed459b2c4d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/17e536830159448da66ebb4363e37edb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/06ab6499fe9e4c40917ee5177000d34c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/d55081b9b36f44f9b08c5c3204d31f71, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/11a47b67c4c448808ae8b5caf422d8f5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/60c5d036922e4e30b4b14daaaa3cfa19, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/59b56f085dcb4e31bd2aca390fe9c112, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a6e88666efda4871863babe485c810cd, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f814413559eb4fa193ae75ed239ecaca, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8f9dc19c231145ccbfef317bc9edf23e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/4702861d11eb4bd6918ec81c74cb6c47, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f227cc0f24644467a3caf3e311fc9188, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/308b7ee89f8c472dbd0e3dea8bf9fa77, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/138875657bd1474a82cdd1e2723abdeb, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/cbc8c2fd47da4b3fa44070b76da5be61, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/2863e01dbe634992aeb170ca34c70503, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/040a8b5a625b4a1eab9d4c460253defc, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/c9267899eaee43c78ecf1ba9f506f131, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/5965edf3eb67423a910d6318e0addabe, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/356c31deb5e147189441a4dee55c4658, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a419759ce2e64e8c812331c7d5a98a99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/7f72baf05bda40c4951662e1248aa0da, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a692db1471ef49059dfa489c1093cb9f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/12c7e00a727f4829be3781c42b9fac04] to archive 2024-11-24T02:48:38,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:48:38,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0d528b8a31674907bc4eb92704f0b83a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0d528b8a31674907bc4eb92704f0b83a 2024-11-24T02:48:38,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/27e63cb180a04225ba490ad04ab1bffd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/27e63cb180a04225ba490ad04ab1bffd 2024-11-24T02:48:38,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8c8ccf6c5fcb410a9e9f414398d6084c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8c8ccf6c5fcb410a9e9f414398d6084c 2024-11-24T02:48:38,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/ae9772cc41c449e68c3618da07128046 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/ae9772cc41c449e68c3618da07128046 2024-11-24T02:48:38,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/1f500698d95b4bedb843792882939648 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/1f500698d95b4bedb843792882939648 2024-11-24T02:48:38,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a8004b6d59a1460b8f983fed459b2c4d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a8004b6d59a1460b8f983fed459b2c4d 2024-11-24T02:48:38,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/17e536830159448da66ebb4363e37edb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/17e536830159448da66ebb4363e37edb 2024-11-24T02:48:38,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/06ab6499fe9e4c40917ee5177000d34c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/06ab6499fe9e4c40917ee5177000d34c 2024-11-24T02:48:38,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/d55081b9b36f44f9b08c5c3204d31f71 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/d55081b9b36f44f9b08c5c3204d31f71 2024-11-24T02:48:38,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/11a47b67c4c448808ae8b5caf422d8f5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/11a47b67c4c448808ae8b5caf422d8f5 2024-11-24T02:48:38,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/60c5d036922e4e30b4b14daaaa3cfa19 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/60c5d036922e4e30b4b14daaaa3cfa19 2024-11-24T02:48:38,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/59b56f085dcb4e31bd2aca390fe9c112 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/59b56f085dcb4e31bd2aca390fe9c112 2024-11-24T02:48:38,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a6e88666efda4871863babe485c810cd to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a6e88666efda4871863babe485c810cd 2024-11-24T02:48:38,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f814413559eb4fa193ae75ed239ecaca to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f814413559eb4fa193ae75ed239ecaca 2024-11-24T02:48:38,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8f9dc19c231145ccbfef317bc9edf23e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/8f9dc19c231145ccbfef317bc9edf23e 2024-11-24T02:48:38,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/4702861d11eb4bd6918ec81c74cb6c47 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/4702861d11eb4bd6918ec81c74cb6c47 2024-11-24T02:48:38,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f227cc0f24644467a3caf3e311fc9188 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/f227cc0f24644467a3caf3e311fc9188 2024-11-24T02:48:38,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/308b7ee89f8c472dbd0e3dea8bf9fa77 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/308b7ee89f8c472dbd0e3dea8bf9fa77 2024-11-24T02:48:38,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/138875657bd1474a82cdd1e2723abdeb to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/138875657bd1474a82cdd1e2723abdeb 2024-11-24T02:48:38,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/cbc8c2fd47da4b3fa44070b76da5be61 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/cbc8c2fd47da4b3fa44070b76da5be61 2024-11-24T02:48:38,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/2863e01dbe634992aeb170ca34c70503 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/2863e01dbe634992aeb170ca34c70503 2024-11-24T02:48:38,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/040a8b5a625b4a1eab9d4c460253defc to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/040a8b5a625b4a1eab9d4c460253defc 2024-11-24T02:48:38,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/c9267899eaee43c78ecf1ba9f506f131 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/c9267899eaee43c78ecf1ba9f506f131 2024-11-24T02:48:38,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/5965edf3eb67423a910d6318e0addabe to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/5965edf3eb67423a910d6318e0addabe 2024-11-24T02:48:38,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/356c31deb5e147189441a4dee55c4658 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/356c31deb5e147189441a4dee55c4658 2024-11-24T02:48:38,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a419759ce2e64e8c812331c7d5a98a99 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a419759ce2e64e8c812331c7d5a98a99 2024-11-24T02:48:38,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/7f72baf05bda40c4951662e1248aa0da to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/7f72baf05bda40c4951662e1248aa0da 2024-11-24T02:48:38,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a692db1471ef49059dfa489c1093cb9f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/a692db1471ef49059dfa489c1093cb9f 2024-11-24T02:48:38,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/12c7e00a727f4829be3781c42b9fac04 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/12c7e00a727f4829be3781c42b9fac04 2024-11-24T02:48:38,759 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/recovered.edits/452.seqid, newMaxSeqId=452, maxSeqId=1 2024-11-24T02:48:38,760 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. 2024-11-24T02:48:38,760 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 5b9e19b406087483d14cf3e721c5b26c: Waiting for close lock at 1732416517400Running coprocessor pre-close hooks at 1732416517400Disabling compacts and flushes for region at 1732416517400Disabling writes for close at 1732416517400Obtaining lock to block concurrent updates at 1732416517401 (+1 ms)Preparing flush snapshotting stores in 5b9e19b406087483d14cf3e721c5b26c at 1732416517401Finished memstore snapshotting TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732416517401Flushing stores of TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c. at 1732416517403 (+2 ms)Flushing 5b9e19b406087483d14cf3e721c5b26c/A: creating writer at 1732416517403Flushing 5b9e19b406087483d14cf3e721c5b26c/A: appending metadata at 1732416517409 (+6 ms)Flushing 5b9e19b406087483d14cf3e721c5b26c/A: closing flushed file at 1732416517409Flushing 5b9e19b406087483d14cf3e721c5b26c/B: creating writer at 1732416517822 (+413 ms)Flushing 5b9e19b406087483d14cf3e721c5b26c/B: appending metadata at 1732416517828 (+6 ms)Flushing 5b9e19b406087483d14cf3e721c5b26c/B: closing flushed file at 1732416517828Flushing 5b9e19b406087483d14cf3e721c5b26c/C: creating writer at 1732416518235 (+407 ms)Flushing 5b9e19b406087483d14cf3e721c5b26c/C: appending metadata at 1732416518238 (+3 ms)Flushing 5b9e19b406087483d14cf3e721c5b26c/C: closing flushed file at 1732416518238Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b629e68: reopening flushed file at 1732416518644 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d769e5: reopening flushed file at 1732416518648 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e3a4d3: reopening flushed file at 1732416518651 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5b9e19b406087483d14cf3e721c5b26c in 1255ms, sequenceid=449, compaction requested=true at 1732416518655 (+4 ms)Writing region close event to WAL at 1732416518756 (+101 ms)Running coprocessor post-close hooks at 1732416518760 (+4 ms)Closed at 1732416518760 2024-11-24T02:48:38,763 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:38,764 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=5b9e19b406087483d14cf3e721c5b26c, regionState=CLOSED 2024-11-24T02:48:38,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:48:38,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-24T02:48:38,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 5b9e19b406087483d14cf3e721c5b26c, server=7c69a60bd8f6,39461,1732416369827 in 1.5200 sec 2024-11-24T02:48:38,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-24T02:48:38,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b9e19b406087483d14cf3e721c5b26c, UNASSIGN in 1.5260 sec 2024-11-24T02:48:38,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-24T02:48:38,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-24T02:48:38,777 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416518777"}]},"ts":"1732416518777"} 2024-11-24T02:48:38,779 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:48:38,779 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:48:38,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5480 sec 2024-11-24T02:48:39,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T02:48:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T02:48:39,387 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:39,387 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:48:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:39,389 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=149, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-24T02:48:39,390 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=149, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:39,391 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:39,394 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/recovered.edits] 2024-11-24T02:48:39,396 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/028460835be2418799a293f2d351f68e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/028460835be2418799a293f2d351f68e 2024-11-24T02:48:39,397 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/1947fcb923ed4d67b4329d2db322d8e9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/1947fcb923ed4d67b4329d2db322d8e9 2024-11-24T02:48:39,398 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/52a51290415e4c67924a4cefcca36806 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/A/52a51290415e4c67924a4cefcca36806 2024-11-24T02:48:39,399 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/b9de4483b866435b915d7f335709f59d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/b9de4483b866435b915d7f335709f59d 2024-11-24T02:48:39,400 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/bef28d84e1ae475fa43c625a541433e7 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/bef28d84e1ae475fa43c625a541433e7 2024-11-24T02:48:39,401 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c517018b78e6478482f72910cda73d79 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/B/c517018b78e6478482f72910cda73d79 2024-11-24T02:48:39,403 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0682499bfb2246edbd9fcd68ffb76a4a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/0682499bfb2246edbd9fcd68ffb76a4a 2024-11-24T02:48:39,404 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/91f49fdfced44f73a086634fa17e47c3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/91f49fdfced44f73a086634fa17e47c3 2024-11-24T02:48:39,405 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/de823a8a6a2a4f368f99cb799ef1a2ef to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/C/de823a8a6a2a4f368f99cb799ef1a2ef 2024-11-24T02:48:39,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/recovered.edits/452.seqid to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c/recovered.edits/452.seqid 2024-11-24T02:48:39,408 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/5b9e19b406087483d14cf3e721c5b26c 2024-11-24T02:48:39,408 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:48:39,410 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=149, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:39,411 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:48:39,414 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:48:39,415 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=149, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:39,415 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:48:39,415 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416519415"}]},"ts":"9223372036854775807"} 2024-11-24T02:48:39,418 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:48:39,418 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5b9e19b406087483d14cf3e721c5b26c, NAME => 'TestAcidGuarantees,,1732416491736.5b9e19b406087483d14cf3e721c5b26c.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:48:39,418 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:48:39,418 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416519418"}]},"ts":"9223372036854775807"} 2024-11-24T02:48:39,420 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:48:39,421 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=149, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:39,422 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-24T02:48:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-24T02:48:39,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:48:39,499 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:39,505 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241 (was 241), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=516 (was 552), ProcessCount=11 (was 11), AvailableMemoryMB=9919 (was 10120) 2024-11-24T02:48:39,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=516, ProcessCount=11, AvailableMemoryMB=9919 2024-11-24T02:48:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:48:39,516 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:48:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:39,518 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T02:48:39,518 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:39,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 150 2024-11-24T02:48:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:48:39,519 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T02:48:39,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742379_1555 (size=959) 2024-11-24T02:48:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:48:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:48:39,929 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3 2024-11-24T02:48:39,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742380_1556 (size=53) 2024-11-24T02:48:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:48:40,335 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:48:40,335 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 824d61e10dc42fc1ca4048b972dd9659, disabling compactions & flushes 2024-11-24T02:48:40,335 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:40,335 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:40,335 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. after waiting 0 ms 2024-11-24T02:48:40,335 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:40,335 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:40,336 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 824d61e10dc42fc1ca4048b972dd9659: Waiting for close lock at 1732416520335Disabling compacts and flushes for region at 1732416520335Disabling writes for close at 1732416520335Writing region close event to WAL at 1732416520335Closed at 1732416520335 2024-11-24T02:48:40,337 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T02:48:40,337 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732416520337"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732416520337"}]},"ts":"1732416520337"} 2024-11-24T02:48:40,338 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T02:48:40,339 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T02:48:40,339 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416520339"}]},"ts":"1732416520339"} 2024-11-24T02:48:40,341 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T02:48:40,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, ASSIGN}] 2024-11-24T02:48:40,342 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, ASSIGN 2024-11-24T02:48:40,342 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, ASSIGN; state=OFFLINE, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=false 2024-11-24T02:48:40,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:40,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, ASSIGN because future has completed 2024-11-24T02:48:40,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:48:40,649 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:40,649 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7752): Opening region: {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:48:40,649 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,649 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:48:40,649 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7794): checking encryption for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,649 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(7797): checking classloading for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,651 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,652 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 824d61e10dc42fc1ca4048b972dd9659 columnFamilyName A 2024-11-24T02:48:40,652 DEBUG [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:40,652 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(327): Store=824d61e10dc42fc1ca4048b972dd9659/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:40,653 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,654 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 824d61e10dc42fc1ca4048b972dd9659 columnFamilyName B 2024-11-24T02:48:40,654 DEBUG [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:40,654 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(327): Store=824d61e10dc42fc1ca4048b972dd9659/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:40,654 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,656 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 824d61e10dc42fc1ca4048b972dd9659 columnFamilyName C 2024-11-24T02:48:40,656 DEBUG [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:40,656 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(327): Store=824d61e10dc42fc1ca4048b972dd9659/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:40,656 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1038): replaying wal for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:48:40,657 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,658 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,659 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1048): stopping wal replay for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,659 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1060): Cleaning up temporary data for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,660 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:48:40,661 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1093): writing seq id for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,663 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:48:40,664 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1114): Opened 824d61e10dc42fc1ca4048b972dd9659; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72247947, jitterRate=0.0765783041715622}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:48:40,664 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:40,665 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegion(1006): Region open journal for 824d61e10dc42fc1ca4048b972dd9659: Running coprocessor pre-open hook at 1732416520649Writing region info on filesystem at 1732416520649Initializing all the Stores at 1732416520650 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416520650Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416520650Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416520650Cleaning up temporary data from old regions at 1732416520659 (+9 ms)Running coprocessor post-open hooks at 1732416520664 (+5 ms)Region opened successfully at 1732416520665 (+1 ms) 2024-11-24T02:48:40,666 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., pid=152, masterSystemTime=1732416520646 2024-11-24T02:48:40,668 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:40,668 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=152}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:40,669 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=OPEN, openSeqNum=2, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:40,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:48:40,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-24T02:48:40,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 in 176 msec 2024-11-24T02:48:40,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-24T02:48:40,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, ASSIGN in 333 msec 2024-11-24T02:48:40,678 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T02:48:40,679 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416520678"}]},"ts":"1732416520678"} 2024-11-24T02:48:40,680 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T02:48:40,682 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=150, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T02:48:40,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-24T02:48:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-24T02:48:41,666 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:41,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:41,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:41,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:41,668 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddad557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:41,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fadc7b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:41,670 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:41,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:41,673 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T02:48:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c69a60bd8f6,43749,1732416369043 2024-11-24T02:48:41,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4df43616 2024-11-24T02:48:41,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T02:48:41,674 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T02:48:41,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T02:48:41,676 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T02:48:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T02:48:41,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742381_1557 (size=995) 2024-11-24T02:48:42,088 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-24T02:48:42,088 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-24T02:48:42,090 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:48:42,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, REOPEN/MOVE}] 2024-11-24T02:48:42,094 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, REOPEN/MOVE 2024-11-24T02:48:42,095 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, REOPEN/MOVE because future has completed 2024-11-24T02:48:42,097 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:48:42,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:48:42,250 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(122): Close 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,250 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:48:42,250 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1722): Closing 824d61e10dc42fc1ca4048b972dd9659, disabling compactions & flushes 2024-11-24T02:48:42,250 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,250 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,250 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. after waiting 0 ms 2024-11-24T02:48:42,250 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,254 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T02:48:42,254 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,254 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegion(1676): Region close journal for 824d61e10dc42fc1ca4048b972dd9659: Waiting for close lock at 1732416522250Running coprocessor pre-close hooks at 1732416522250Disabling compacts and flushes for region at 1732416522250Disabling writes for close at 1732416522250Writing region close event to WAL at 1732416522251 (+1 ms)Running coprocessor post-close hooks at 1732416522254 (+3 ms)Closed at 1732416522254 2024-11-24T02:48:42,254 WARN [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] regionserver.HRegionServer(3299): Not adding moved region record: 824d61e10dc42fc1ca4048b972dd9659 to self. 2024-11-24T02:48:42,256 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=156}] handler.UnassignRegionHandler(157): Closed 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,256 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=CLOSED 2024-11-24T02:48:42,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; CloseRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:48:42,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-24T02:48:42,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; CloseRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 in 161 msec 2024-11-24T02:48:42,260 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, REOPEN/MOVE; state=CLOSED, location=7c69a60bd8f6,39461,1732416369827; forceNewPlan=false, retain=true 2024-11-24T02:48:42,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=OPENING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, REOPEN/MOVE because future has completed 2024-11-24T02:48:42,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:48:42,566 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,566 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7752): Opening region: {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} 2024-11-24T02:48:42,567 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,567 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:48:42,567 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7794): checking encryption for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,567 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(7797): checking classloading for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,568 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,568 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 824d61e10dc42fc1ca4048b972dd9659 columnFamilyName A 2024-11-24T02:48:42,569 DEBUG [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:42,570 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(327): Store=824d61e10dc42fc1ca4048b972dd9659/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:42,570 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,571 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 824d61e10dc42fc1ca4048b972dd9659 columnFamilyName B 2024-11-24T02:48:42,571 DEBUG [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:42,571 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(327): Store=824d61e10dc42fc1ca4048b972dd9659/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:42,571 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,572 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 824d61e10dc42fc1ca4048b972dd9659 columnFamilyName C 2024-11-24T02:48:42,572 DEBUG [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:42,573 INFO [StoreOpener-824d61e10dc42fc1ca4048b972dd9659-1 {}] regionserver.HStore(327): Store=824d61e10dc42fc1ca4048b972dd9659/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:48:42,573 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1038): replaying wal for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,573 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,574 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,575 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1048): stopping wal replay for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,575 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1060): Cleaning up temporary data for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,575 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T02:48:42,576 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1093): writing seq id for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,577 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1114): Opened 824d61e10dc42fc1ca4048b972dd9659; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73195540, jitterRate=0.09069854021072388}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T02:48:42,577 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,577 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegion(1006): Region open journal for 824d61e10dc42fc1ca4048b972dd9659: Running coprocessor pre-open hook at 1732416522567Writing region info on filesystem at 1732416522567Initializing all the Stores at 1732416522567Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416522567Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416522568 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732416522568Cleaning up temporary data from old regions at 1732416522575 (+7 ms)Running coprocessor post-open hooks at 1732416522577 (+2 ms)Region opened successfully at 1732416522577 2024-11-24T02:48:42,578 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., pid=157, masterSystemTime=1732416522564 2024-11-24T02:48:42,580 DEBUG [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,580 INFO [RS_OPEN_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_OPEN_REGION, pid=157}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,581 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=OPEN, openSeqNum=5, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:48:42,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=155 2024-11-24T02:48:42,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 in 170 msec 2024-11-24T02:48:42,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-24T02:48:42,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, REOPEN/MOVE in 491 msec 2024-11-24T02:48:42,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-24T02:48:42,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 496 msec 2024-11-24T02:48:42,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-11-24T02:48:42,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-24T02:48:42,590 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:42,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3bf8c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55311462, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,595 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fe9814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,596 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c473a4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5573d8cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,599 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@719e12e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,600 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f3dd20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,601 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19582049, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,602 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,603 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@566ff2e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,604 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dafad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,605 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d86066f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,606 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9f86ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,608 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bcc07dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,609 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f0f386e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,610 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,610 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523cd874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,612 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73bc6e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,614 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c7658cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,615 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7401d306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c69a60bd8f6,43749,-1 for getting cluster id 2024-11-24T02:48:42,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T02:48:42,617 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc' 2024-11-24T02:48:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T02:48:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cfa0c6f-4040-42a1-94fd-cd4dcb46c6cc" 2024-11-24T02:48:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e351937, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c69a60bd8f6,43749,-1] 2024-11-24T02:48:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T02:48:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:48:42,618 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T02:48:42,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f340e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T02:48:42,619 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,619 DEBUG [Thread-2488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,619 DEBUG [Thread-2490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,619 DEBUG [Thread-2477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,620 DEBUG [Thread-2481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,620 DEBUG [Thread-2492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,620 DEBUG [Thread-2481 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,621 DEBUG [Thread-2481 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,623 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:42,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-24T02:48:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:48:42,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,626 DEBUG [Thread-2479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,626 DEBUG [Thread-2484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,626 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,628 DEBUG [Thread-2486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,628 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:42,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:42,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,631 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T02:48:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c69a60bd8f6,39461,1732416369827, seqNum=-1] 2024-11-24T02:48:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T02:48:42,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T02:48:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:42,635 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:42,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:48:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,636 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5] 2024-11-24T02:48:42,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e38bdd122a0949d39ce07002a011fc5c_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416522630/Put/seqid=0 2024-11-24T02:48:42,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742382_1558 (size=12154) 2024-11-24T02:48:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416582682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416582683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416582684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416582685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416582686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:48:42,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:48:42,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:42,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:42,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416582797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416582797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416582797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416582797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:42,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416582797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:42,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:42,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:48:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:42,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:42,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:48:43,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416583006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416583006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416583007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416583007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416583007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:43,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e38bdd122a0949d39ce07002a011fc5c_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e38bdd122a0949d39ce07002a011fc5c_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:43,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/e8a690457e524d428afacdf6f4ee83e6, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:43,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/e8a690457e524d428afacdf6f4ee83e6 is 175, key is test_row_0/A:col10/1732416522630/Put/seqid=0 2024-11-24T02:48:43,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742383_1559 (size=30955) 2024-11-24T02:48:43,068 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/e8a690457e524d428afacdf6f4ee83e6 2024-11-24T02:48:43,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:48:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/1409c002dc85494b96e55656a738d9b6 is 50, key is test_row_0/B:col10/1732416522630/Put/seqid=0 2024-11-24T02:48:43,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742384_1560 (size=12001) 2024-11-24T02:48:43,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/1409c002dc85494b96e55656a738d9b6 2024-11-24T02:48:43,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/617198e0abc548b38c4c4188c3cfedac is 50, key is test_row_0/C:col10/1732416522630/Put/seqid=0 2024-11-24T02:48:43,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742385_1561 (size=12001) 2024-11-24T02:48:43,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:48:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:43,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:48:43,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416583317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416583317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416583317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416583317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416583317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:48:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:43,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/617198e0abc548b38c4c4188c3cfedac 2024-11-24T02:48:43,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:48:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/e8a690457e524d428afacdf6f4ee83e6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/e8a690457e524d428afacdf6f4ee83e6 2024-11-24T02:48:43,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/e8a690457e524d428afacdf6f4ee83e6, entries=150, sequenceid=16, filesize=30.2 K 2024-11-24T02:48:43,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/1409c002dc85494b96e55656a738d9b6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/1409c002dc85494b96e55656a738d9b6 2024-11-24T02:48:43,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/1409c002dc85494b96e55656a738d9b6, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T02:48:43,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/617198e0abc548b38c4c4188c3cfedac as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/617198e0abc548b38c4c4188c3cfedac 2024-11-24T02:48:43,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/617198e0abc548b38c4c4188c3cfedac, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T02:48:43,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 824d61e10dc42fc1ca4048b972dd9659 in 939ms, sequenceid=16, compaction requested=false 2024-11-24T02:48:43,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:43,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T02:48:43,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:43,704 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:48:43,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112465cc347de65f45fe842aec9191421ce0_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416522681/Put/seqid=0 2024-11-24T02:48:43,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742386_1562 (size=12154) 2024-11-24T02:48:43,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:48:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:43,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416583832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416583834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416583837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416583838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416583840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416583947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416583947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416583947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416583947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416583947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:44,159 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112465cc347de65f45fe842aec9191421ce0_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112465cc347de65f45fe842aec9191421ce0_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:44,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416584157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/f3e965dafdf642029a846f032e76978f, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416584157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416584157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/f3e965dafdf642029a846f032e76978f is 175, key is test_row_0/A:col10/1732416522681/Put/seqid=0 2024-11-24T02:48:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416584157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416584157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742387_1563 (size=30955) 2024-11-24T02:48:44,193 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/f3e965dafdf642029a846f032e76978f 2024-11-24T02:48:44,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/9f2ecd0e17ef45e6be11a7b0430e9525 is 50, key is test_row_0/B:col10/1732416522681/Put/seqid=0 2024-11-24T02:48:44,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742388_1564 (size=12001) 2024-11-24T02:48:44,238 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/9f2ecd0e17ef45e6be11a7b0430e9525 2024-11-24T02:48:44,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/b8685ee6ae494fd4bf6ce31d6e488ba2 is 50, key is test_row_0/C:col10/1732416522681/Put/seqid=0 2024-11-24T02:48:44,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742389_1565 (size=12001) 2024-11-24T02:48:44,283 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T02:48:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416584466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416584466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416584466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416584467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416584477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:44,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:44,655 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/b8685ee6ae494fd4bf6ce31d6e488ba2 2024-11-24T02:48:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/f3e965dafdf642029a846f032e76978f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/f3e965dafdf642029a846f032e76978f 2024-11-24T02:48:44,661 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/f3e965dafdf642029a846f032e76978f, entries=150, sequenceid=40, filesize=30.2 K 2024-11-24T02:48:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/9f2ecd0e17ef45e6be11a7b0430e9525 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/9f2ecd0e17ef45e6be11a7b0430e9525 2024-11-24T02:48:44,664 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/9f2ecd0e17ef45e6be11a7b0430e9525, entries=150, sequenceid=40, filesize=11.7 K 2024-11-24T02:48:44,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/b8685ee6ae494fd4bf6ce31d6e488ba2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/b8685ee6ae494fd4bf6ce31d6e488ba2 2024-11-24T02:48:44,667 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/b8685ee6ae494fd4bf6ce31d6e488ba2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-24T02:48:44,668 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 824d61e10dc42fc1ca4048b972dd9659 in 964ms, sequenceid=40, compaction requested=false 2024-11-24T02:48:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:44,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-24T02:48:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-24T02:48:44,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-24T02:48:44,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0430 sec 2024-11-24T02:48:44,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.0520 sec 2024-11-24T02:48:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T02:48:44,776 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:44,777 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-24T02:48:44,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:48:44,778 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:44,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:44,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:48:44,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T02:48:44,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:44,931 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:48:44,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e4270a7e3f91478bae1e778f82989b1c_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416523832/Put/seqid=0 2024-11-24T02:48:44,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742390_1566 (size=12154) 2024-11-24T02:48:44,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:44,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416584999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416585003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416585003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416585004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416585004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:48:45,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416585106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416585116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416585117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416585117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416585117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416585316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416585326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416585327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416585327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416585327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:45,343 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e4270a7e3f91478bae1e778f82989b1c_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4270a7e3f91478bae1e778f82989b1c_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:45,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5f9b7eea9d71450faddd3c70e32d8775, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:45,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5f9b7eea9d71450faddd3c70e32d8775 is 175, key is test_row_0/A:col10/1732416523832/Put/seqid=0 2024-11-24T02:48:45,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742391_1567 (size=30955) 2024-11-24T02:48:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:48:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416585626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416585636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416585636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416585636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416585637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:45,747 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5f9b7eea9d71450faddd3c70e32d8775 2024-11-24T02:48:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/326392c656094f19813a6a18a6878ece is 50, key is test_row_0/B:col10/1732416523832/Put/seqid=0 2024-11-24T02:48:45,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742392_1568 (size=12001) 2024-11-24T02:48:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:48:46,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:46,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416586136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:46,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416586146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:46,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416586147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416586147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416586147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:46,162 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/326392c656094f19813a6a18a6878ece 2024-11-24T02:48:46,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/70a2b929952e426e97d9892a9f79ea56 is 50, key is test_row_0/C:col10/1732416523832/Put/seqid=0 2024-11-24T02:48:46,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742393_1569 (size=12001) 2024-11-24T02:48:46,570 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/70a2b929952e426e97d9892a9f79ea56 2024-11-24T02:48:46,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5f9b7eea9d71450faddd3c70e32d8775 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5f9b7eea9d71450faddd3c70e32d8775 2024-11-24T02:48:46,576 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5f9b7eea9d71450faddd3c70e32d8775, entries=150, sequenceid=52, filesize=30.2 K 2024-11-24T02:48:46,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/326392c656094f19813a6a18a6878ece as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/326392c656094f19813a6a18a6878ece 2024-11-24T02:48:46,580 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/326392c656094f19813a6a18a6878ece, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T02:48:46,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/70a2b929952e426e97d9892a9f79ea56 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/70a2b929952e426e97d9892a9f79ea56 2024-11-24T02:48:46,583 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/70a2b929952e426e97d9892a9f79ea56, entries=150, sequenceid=52, filesize=11.7 K 2024-11-24T02:48:46,584 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 824d61e10dc42fc1ca4048b972dd9659 in 1653ms, sequenceid=52, compaction requested=true 2024-11-24T02:48:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-24T02:48:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-24T02:48:46,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-24T02:48:46,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8060 sec 2024-11-24T02:48:46,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8100 sec 2024-11-24T02:48:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T02:48:46,937 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:46,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:46,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-24T02:48:46,938 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:48:46,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:46,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:48:47,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T02:48:47,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:47,091 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:48:47,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124aaa203be9e114413b1c5360f3cd39e29_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416525002/Put/seqid=0 2024-11-24T02:48:47,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742394_1570 (size=12154) 2024-11-24T02:48:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:47,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416587163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416587164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416587165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416587164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416587166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,171 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:48:47,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416587276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416587277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416587277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416587277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416587486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416587486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416587487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416587487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:47,502 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124aaa203be9e114413b1c5360f3cd39e29_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aaa203be9e114413b1c5360f3cd39e29_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:47,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/d0a0da70ae34449292f15ee336dae92a, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:47,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/d0a0da70ae34449292f15ee336dae92a is 175, key is test_row_0/A:col10/1732416525002/Put/seqid=0 2024-11-24T02:48:47,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742395_1571 (size=30955) 2024-11-24T02:48:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:48:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416587796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416587796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416587796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416587796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:47,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:47,906 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/d0a0da70ae34449292f15ee336dae92a 2024-11-24T02:48:47,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/5e799dbba36048909454bd5515806e62 is 50, key is test_row_0/B:col10/1732416525002/Put/seqid=0 2024-11-24T02:48:47,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742396_1572 (size=12001) 2024-11-24T02:48:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:48:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416588306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416588306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416588306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416588306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:48,316 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/5e799dbba36048909454bd5515806e62 2024-11-24T02:48:48,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/ee846bc2bd29489baea4319802155304 is 50, key is test_row_0/C:col10/1732416525002/Put/seqid=0 2024-11-24T02:48:48,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742397_1573 (size=12001) 2024-11-24T02:48:48,725 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/ee846bc2bd29489baea4319802155304 2024-11-24T02:48:48,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/d0a0da70ae34449292f15ee336dae92a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/d0a0da70ae34449292f15ee336dae92a 2024-11-24T02:48:48,731 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/d0a0da70ae34449292f15ee336dae92a, entries=150, sequenceid=77, filesize=30.2 K 2024-11-24T02:48:48,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/5e799dbba36048909454bd5515806e62 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/5e799dbba36048909454bd5515806e62 2024-11-24T02:48:48,734 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/5e799dbba36048909454bd5515806e62, entries=150, sequenceid=77, filesize=11.7 K 2024-11-24T02:48:48,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/ee846bc2bd29489baea4319802155304 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/ee846bc2bd29489baea4319802155304 2024-11-24T02:48:48,737 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/ee846bc2bd29489baea4319802155304, entries=150, sequenceid=77, filesize=11.7 K 2024-11-24T02:48:48,738 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 824d61e10dc42fc1ca4048b972dd9659 in 1647ms, sequenceid=77, compaction requested=true 2024-11-24T02:48:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-24T02:48:48,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-24T02:48:48,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-24T02:48:48,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-24T02:48:48,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8040 sec 2024-11-24T02:48:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T02:48:49,086 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:49,087 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:49,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-24T02:48:49,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:48:49,088 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:49,089 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:49,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:49,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-24T02:48:49,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112480be143cd8c94f9fb7a98d6170bf1862_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416527163/Put/seqid=0 2024-11-24T02:48:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:48:49,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742398_1574 (size=12154) 2024-11-24T02:48:49,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:49,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416589290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416589316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416589316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,317 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416589316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,318 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:49,318 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:49,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416589326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,330 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:49,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T02:48:49,374 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T02:48:49,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:49,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:49,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:48:49,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416589406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:49,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:49,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112480be143cd8c94f9fb7a98d6170bf1862_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112480be143cd8c94f9fb7a98d6170bf1862_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:49,604 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/2b9326feaa9c4fd58a0a8145e587c823, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:49,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/2b9326feaa9c4fd58a0a8145e587c823 is 175, key is test_row_0/A:col10/1732416527163/Put/seqid=0 2024-11-24T02:48:49,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742399_1575 (size=30955) 2024-11-24T02:48:49,607 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/2b9326feaa9c4fd58a0a8145e587c823 2024-11-24T02:48:49,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/d6a0dc327dc44554948a5930de4977ce is 50, key is test_row_0/B:col10/1732416527163/Put/seqid=0 2024-11-24T02:48:49,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742400_1576 (size=12001) 2024-11-24T02:48:49,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416589617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:49,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:48:49,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:49,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:49,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416589926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:50,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:50,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/d6a0dc327dc44554948a5930de4977ce 2024-11-24T02:48:50,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/bce4fb2589ff4f50bf2302407e105699 is 50, key is test_row_0/C:col10/1732416527163/Put/seqid=0 2024-11-24T02:48:50,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742401_1577 (size=12001) 2024-11-24T02:48:50,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:48:50,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/bce4fb2589ff4f50bf2302407e105699 2024-11-24T02:48:50,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/2b9326feaa9c4fd58a0a8145e587c823 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/2b9326feaa9c4fd58a0a8145e587c823 2024-11-24T02:48:50,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/2b9326feaa9c4fd58a0a8145e587c823, entries=150, sequenceid=89, filesize=30.2 K 2024-11-24T02:48:50,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/d6a0dc327dc44554948a5930de4977ce as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/d6a0dc327dc44554948a5930de4977ce 2024-11-24T02:48:50,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/d6a0dc327dc44554948a5930de4977ce, entries=150, sequenceid=89, filesize=11.7 K 2024-11-24T02:48:50,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/bce4fb2589ff4f50bf2302407e105699 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/bce4fb2589ff4f50bf2302407e105699 2024-11-24T02:48:50,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/bce4fb2589ff4f50bf2302407e105699, entries=150, sequenceid=89, filesize=11.7 K 2024-11-24T02:48:50,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 824d61e10dc42fc1ca4048b972dd9659 in 1248ms, sequenceid=89, compaction requested=true 2024-11-24T02:48:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:50,437 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T02:48:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:50,437 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T02:48:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:50,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:50,439 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 154775 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T02:48:50,439 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T02:48:50,439 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/B is initiating minor compaction (all files) 2024-11-24T02:48:50,439 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/B in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,439 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/1409c002dc85494b96e55656a738d9b6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/9f2ecd0e17ef45e6be11a7b0430e9525, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/326392c656094f19813a6a18a6878ece, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/5e799dbba36048909454bd5515806e62, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/d6a0dc327dc44554948a5930de4977ce] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=58.6 K 2024-11-24T02:48:50,440 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/A is initiating minor compaction (all files) 2024-11-24T02:48:50,440 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/A in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,440 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/e8a690457e524d428afacdf6f4ee83e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/f3e965dafdf642029a846f032e76978f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5f9b7eea9d71450faddd3c70e32d8775, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/d0a0da70ae34449292f15ee336dae92a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/2b9326feaa9c4fd58a0a8145e587c823] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=151.1 K 2024-11-24T02:48:50,440 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,440 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/e8a690457e524d428afacdf6f4ee83e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/f3e965dafdf642029a846f032e76978f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5f9b7eea9d71450faddd3c70e32d8775, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/d0a0da70ae34449292f15ee336dae92a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/2b9326feaa9c4fd58a0a8145e587c823] 2024-11-24T02:48:50,440 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1409c002dc85494b96e55656a738d9b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732416522630 2024-11-24T02:48:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:50,440 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:50,440 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a690457e524d428afacdf6f4ee83e6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732416522630 2024-11-24T02:48:50,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:48:50,440 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f2ecd0e17ef45e6be11a7b0430e9525, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732416522681 2024-11-24T02:48:50,441 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3e965dafdf642029a846f032e76978f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732416522681 2024-11-24T02:48:50,441 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 326392c656094f19813a6a18a6878ece, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416523832 2024-11-24T02:48:50,441 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f9b7eea9d71450faddd3c70e32d8775, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416523832 2024-11-24T02:48:50,441 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e799dbba36048909454bd5515806e62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416525002 2024-11-24T02:48:50,441 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a0dc327dc44554948a5930de4977ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732416527159 2024-11-24T02:48:50,441 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a0da70ae34449292f15ee336dae92a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416525002 2024-11-24T02:48:50,441 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9326feaa9c4fd58a0a8145e587c823, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732416527159 2024-11-24T02:48:50,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249ef83ab6a7ac4efd866ccb214f662d4c_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416529288/Put/seqid=0 2024-11-24T02:48:50,451 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#B#compaction#492 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:50,451 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/996b4b73be0d4d9a860587644d4d3c5c is 50, key is test_row_0/B:col10/1732416527163/Put/seqid=0 2024-11-24T02:48:50,453 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:50,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742402_1578 (size=14594) 2024-11-24T02:48:50,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,468 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124b170200891a440beac53a474b72ba40d_824d61e10dc42fc1ca4048b972dd9659 store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:50,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742403_1579 (size=12173) 2024-11-24T02:48:50,470 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124b170200891a440beac53a474b72ba40d_824d61e10dc42fc1ca4048b972dd9659, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:50,470 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b170200891a440beac53a474b72ba40d_824d61e10dc42fc1ca4048b972dd9659 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:50,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742404_1580 (size=4469) 2024-11-24T02:48:50,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416590488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:50,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:50,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416590596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:50,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:50,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:50,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:50,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:50,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416590807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:50,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:50,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:50,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:50,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249ef83ab6a7ac4efd866ccb214f662d4c_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249ef83ab6a7ac4efd866ccb214f662d4c_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:50,865 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/4357555ea64947f9bd63279a36487a73, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:50,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/4357555ea64947f9bd63279a36487a73 is 175, key is test_row_0/A:col10/1732416529288/Put/seqid=0 2024-11-24T02:48:50,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742405_1581 (size=39549) 2024-11-24T02:48:50,872 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/996b4b73be0d4d9a860587644d4d3c5c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/996b4b73be0d4d9a860587644d4d3c5c 2024-11-24T02:48:50,874 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/4357555ea64947f9bd63279a36487a73 2024-11-24T02:48:50,876 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#A#compaction#493 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:50,876 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/c495ff67843a4bdaa018ea6988d631b4 is 175, key is test_row_0/A:col10/1732416527163/Put/seqid=0 2024-11-24T02:48:50,877 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/B of 824d61e10dc42fc1ca4048b972dd9659 into 996b4b73be0d4d9a860587644d4d3c5c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:50,877 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:50,877 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/B, priority=11, startTime=1732416530437; duration=0sec 2024-11-24T02:48:50,877 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:50,877 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:B 2024-11-24T02:48:50,877 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-24T02:48:50,879 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-24T02:48:50,879 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/C is initiating minor compaction (all files) 2024-11-24T02:48:50,879 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/C in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,879 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/617198e0abc548b38c4c4188c3cfedac, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/b8685ee6ae494fd4bf6ce31d6e488ba2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/70a2b929952e426e97d9892a9f79ea56, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/ee846bc2bd29489baea4319802155304, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/bce4fb2589ff4f50bf2302407e105699] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=58.6 K 2024-11-24T02:48:50,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742406_1582 (size=31127) 2024-11-24T02:48:50,880 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 617198e0abc548b38c4c4188c3cfedac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732416522630 2024-11-24T02:48:50,880 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b8685ee6ae494fd4bf6ce31d6e488ba2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732416522681 2024-11-24T02:48:50,880 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 70a2b929952e426e97d9892a9f79ea56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732416523832 2024-11-24T02:48:50,881 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting ee846bc2bd29489baea4319802155304, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732416525002 2024-11-24T02:48:50,881 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting bce4fb2589ff4f50bf2302407e105699, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732416527159 2024-11-24T02:48:50,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/08dbd0829bbb4b73aa5ead61a9529369 is 50, key is test_row_0/B:col10/1732416529288/Put/seqid=0 2024-11-24T02:48:50,885 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/c495ff67843a4bdaa018ea6988d631b4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/c495ff67843a4bdaa018ea6988d631b4 2024-11-24T02:48:50,888 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/A of 824d61e10dc42fc1ca4048b972dd9659 into c495ff67843a4bdaa018ea6988d631b4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:50,889 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:50,889 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/A, priority=11, startTime=1732416530437; duration=0sec 2024-11-24T02:48:50,889 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:50,889 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:A 2024-11-24T02:48:50,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742407_1583 (size=12001) 2024-11-24T02:48:50,892 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#C#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:50,892 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/0fed06312a034e18be067b31c6f314a9 is 50, key is test_row_0/C:col10/1732416527163/Put/seqid=0 2024-11-24T02:48:50,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742408_1584 (size=12173) 2024-11-24T02:48:50,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/08dbd0829bbb4b73aa5ead61a9529369 2024-11-24T02:48:50,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/21a966444d594860893133d17f15672c is 50, key is test_row_0/C:col10/1732416529288/Put/seqid=0 2024-11-24T02:48:50,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742409_1585 (size=12001) 2024-11-24T02:48:50,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/21a966444d594860893133d17f15672c 2024-11-24T02:48:50,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/4357555ea64947f9bd63279a36487a73 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/4357555ea64947f9bd63279a36487a73 2024-11-24T02:48:50,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/4357555ea64947f9bd63279a36487a73, entries=200, sequenceid=114, filesize=38.6 K 2024-11-24T02:48:50,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/08dbd0829bbb4b73aa5ead61a9529369 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/08dbd0829bbb4b73aa5ead61a9529369 2024-11-24T02:48:50,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/08dbd0829bbb4b73aa5ead61a9529369, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:48:50,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/21a966444d594860893133d17f15672c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/21a966444d594860893133d17f15672c 2024-11-24T02:48:50,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/21a966444d594860893133d17f15672c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T02:48:50,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 824d61e10dc42fc1ca4048b972dd9659 in 479ms, sequenceid=114, compaction requested=false 2024-11-24T02:48:50,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:50,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-24T02:48:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:50,925 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:48:50,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249873efe5075049f284e9dc91afdc3212_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416530486/Put/seqid=0 2024-11-24T02:48:50,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742410_1586 (size=12154) 2024-11-24T02:48:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:51,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:51,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416591208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:48:51,298 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/0fed06312a034e18be067b31c6f314a9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/0fed06312a034e18be067b31c6f314a9 2024-11-24T02:48:51,302 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/C of 824d61e10dc42fc1ca4048b972dd9659 into 0fed06312a034e18be067b31c6f314a9(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:51,302 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:51,302 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/C, priority=11, startTime=1732416530437; duration=0sec 2024-11-24T02:48:51,302 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:51,302 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:C 2024-11-24T02:48:51,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416591317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:51,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:51,337 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249873efe5075049f284e9dc91afdc3212_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249873efe5075049f284e9dc91afdc3212_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:51,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/545e4cad89e3437bb3d4879d04b30bd8, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:51,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/545e4cad89e3437bb3d4879d04b30bd8 is 175, key is test_row_0/A:col10/1732416530486/Put/seqid=0 2024-11-24T02:48:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416591336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416591336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,339 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416591336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:51,339 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:51,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742411_1587 (size=30955) 2024-11-24T02:48:51,341 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:51,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416591357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,359 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_2' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:51,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416591527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:51,742 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/545e4cad89e3437bb3d4879d04b30bd8 2024-11-24T02:48:51,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/daba5f1de96446c49fa7c37c5682a13d is 50, key is test_row_0/B:col10/1732416530486/Put/seqid=0 2024-11-24T02:48:51,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742412_1588 (size=12001) 2024-11-24T02:48:51,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416591836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:51,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:52,151 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/daba5f1de96446c49fa7c37c5682a13d 2024-11-24T02:48:52,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/82803e35f54b42dc964f143f0f9a760b is 50, key is test_row_0/C:col10/1732416530486/Put/seqid=0 2024-11-24T02:48:52,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742413_1589 (size=12001) 2024-11-24T02:48:52,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416592347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:52,612 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/82803e35f54b42dc964f143f0f9a760b 2024-11-24T02:48:52,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/545e4cad89e3437bb3d4879d04b30bd8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/545e4cad89e3437bb3d4879d04b30bd8 2024-11-24T02:48:52,619 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/545e4cad89e3437bb3d4879d04b30bd8, entries=150, sequenceid=127, filesize=30.2 K 2024-11-24T02:48:52,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/daba5f1de96446c49fa7c37c5682a13d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/daba5f1de96446c49fa7c37c5682a13d 2024-11-24T02:48:52,622 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/daba5f1de96446c49fa7c37c5682a13d, entries=150, sequenceid=127, filesize=11.7 K 2024-11-24T02:48:52,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/82803e35f54b42dc964f143f0f9a760b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/82803e35f54b42dc964f143f0f9a760b 2024-11-24T02:48:52,625 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/82803e35f54b42dc964f143f0f9a760b, entries=150, sequenceid=127, filesize=11.7 K 2024-11-24T02:48:52,625 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 824d61e10dc42fc1ca4048b972dd9659 in 1700ms, sequenceid=127, compaction requested=true 2024-11-24T02:48:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-24T02:48:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-24T02:48:52,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-24T02:48:52,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5370 sec 2024-11-24T02:48:52,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 3.5410 sec 2024-11-24T02:48:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T02:48:53,247 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:53,247 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-24T02:48:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T02:48:53,249 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:53,249 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:53,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T02:48:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:53,359 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:53,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-24T02:48:53,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124769ed39f2a3e48a48ad53a8fe9adfaeb_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416531207/Put/seqid=0 2024-11-24T02:48:53,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742414_1590 (size=14794) 2024-11-24T02:48:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416593393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:53,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:53,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416593506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:53,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:53,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T02:48:53,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:53,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416593716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:53,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:53,784 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:53,787 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124769ed39f2a3e48a48ad53a8fe9adfaeb_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124769ed39f2a3e48a48ad53a8fe9adfaeb_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:53,787 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/9048161e71ce4b3c954b30e771e30f06, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:53,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/9048161e71ce4b3c954b30e771e30f06 is 175, key is test_row_0/A:col10/1732416531207/Put/seqid=0 2024-11-24T02:48:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742415_1591 (size=39749) 2024-11-24T02:48:53,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:53,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:53,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T02:48:54,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416594027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:54,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:54,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,191 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/9048161e71ce4b3c954b30e771e30f06 2024-11-24T02:48:54,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/4d26593138a441a1804360f03986df83 is 50, key is test_row_0/B:col10/1732416531207/Put/seqid=0 2024-11-24T02:48:54,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742416_1592 (size=12151) 2024-11-24T02:48:54,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T02:48:54,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416594546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:54,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/4d26593138a441a1804360f03986df83 2024-11-24T02:48:54,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/473ec0a24f7a4d7f99d19437fd229672 is 50, key is test_row_0/C:col10/1732416531207/Put/seqid=0 2024-11-24T02:48:54,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:48:54,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742417_1593 (size=12151) 2024-11-24T02:48:54,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/473ec0a24f7a4d7f99d19437fd229672 2024-11-24T02:48:54,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/9048161e71ce4b3c954b30e771e30f06 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9048161e71ce4b3c954b30e771e30f06 2024-11-24T02:48:54,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9048161e71ce4b3c954b30e771e30f06, entries=200, sequenceid=154, filesize=38.8 K 2024-11-24T02:48:54,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/4d26593138a441a1804360f03986df83 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/4d26593138a441a1804360f03986df83 2024-11-24T02:48:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/4d26593138a441a1804360f03986df83, entries=150, sequenceid=154, filesize=11.9 K 2024-11-24T02:48:54,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/473ec0a24f7a4d7f99d19437fd229672 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/473ec0a24f7a4d7f99d19437fd229672 2024-11-24T02:48:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/473ec0a24f7a4d7f99d19437fd229672, entries=150, sequenceid=154, filesize=11.9 K 2024-11-24T02:48:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 824d61e10dc42fc1ca4048b972dd9659 in 1319ms, sequenceid=154, compaction requested=true 2024-11-24T02:48:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:54,678 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:54,679 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:54,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:54,681 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,681 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/B is initiating minor compaction (all files) 2024-11-24T02:48:54,682 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/B in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,682 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/996b4b73be0d4d9a860587644d4d3c5c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/08dbd0829bbb4b73aa5ead61a9529369, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/daba5f1de96446c49fa7c37c5682a13d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/4d26593138a441a1804360f03986df83] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=47.2 K 2024-11-24T02:48:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,682 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 996b4b73be0d4d9a860587644d4d3c5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732416527159 2024-11-24T02:48:54,682 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 08dbd0829bbb4b73aa5ead61a9529369, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416529281 2024-11-24T02:48:54,683 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting daba5f1de96446c49fa7c37c5682a13d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732416530455 2024-11-24T02:48:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,683 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141380 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:54,683 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/A is initiating minor compaction (all files) 2024-11-24T02:48:54,683 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/A in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,683 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/c495ff67843a4bdaa018ea6988d631b4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/4357555ea64947f9bd63279a36487a73, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/545e4cad89e3437bb3d4879d04b30bd8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9048161e71ce4b3c954b30e771e30f06] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=138.1 K 2024-11-24T02:48:54,683 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,683 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/c495ff67843a4bdaa018ea6988d631b4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/4357555ea64947f9bd63279a36487a73, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/545e4cad89e3437bb3d4879d04b30bd8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9048161e71ce4b3c954b30e771e30f06] 2024-11-24T02:48:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,684 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d26593138a441a1804360f03986df83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416531187 2024-11-24T02:48:54,684 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c495ff67843a4bdaa018ea6988d631b4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732416527159 2024-11-24T02:48:54,684 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4357555ea64947f9bd63279a36487a73, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416529256 2024-11-24T02:48:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,685 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 545e4cad89e3437bb3d4879d04b30bd8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732416530455 2024-11-24T02:48:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,686 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9048161e71ce4b3c954b30e771e30f06, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416531187 2024-11-24T02:48:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,705 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,715 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:54,715 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/b3d9cda38f86415b8c73abc80b319719 is 50, key is test_row_0/B:col10/1732416531207/Put/seqid=0 2024-11-24T02:48:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,726 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124d34ac71278764f5bab903212327591b8_824d61e10dc42fc1ca4048b972dd9659 store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,728 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124d34ac71278764f5bab903212327591b8_824d61e10dc42fc1ca4048b972dd9659, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,729 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d34ac71278764f5bab903212327591b8_824d61e10dc42fc1ca4048b972dd9659 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742419_1595 (size=4469) 2024-11-24T02:48:54,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742418_1594 (size=12459) 2024-11-24T02:48:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,776 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/b3d9cda38f86415b8c73abc80b319719 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/b3d9cda38f86415b8c73abc80b319719 2024-11-24T02:48:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,781 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/B of 824d61e10dc42fc1ca4048b972dd9659 into b3d9cda38f86415b8c73abc80b319719(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:54,781 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:54,781 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/B, priority=12, startTime=1732416534679; duration=0sec 2024-11-24T02:48:54,781 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:54,781 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:B 2024-11-24T02:48:54,781 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:48:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,782 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:48:54,783 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/C is initiating minor compaction (all files) 2024-11-24T02:48:54,783 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/C in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,783 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/0fed06312a034e18be067b31c6f314a9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/21a966444d594860893133d17f15672c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/82803e35f54b42dc964f143f0f9a760b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/473ec0a24f7a4d7f99d19437fd229672] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=47.2 K 2024-11-24T02:48:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,783 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fed06312a034e18be067b31c6f314a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732416527159 2024-11-24T02:48:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,783 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a966444d594860893133d17f15672c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732416529281 2024-11-24T02:48:54,784 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 82803e35f54b42dc964f143f0f9a760b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732416530455 2024-11-24T02:48:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,784 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 473ec0a24f7a4d7f99d19437fd229672, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416531187 2024-11-24T02:48:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-24T02:48:54,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:54,785 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-24T02:48:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,796 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#C#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:54,797 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/3a6befabe0294df59f48ce6677931db5 is 50, key is test_row_0/C:col10/1732416531207/Put/seqid=0 2024-11-24T02:48:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112434fad882d07a455aa543dc45d53c77c5_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416533385/Put/seqid=0 2024-11-24T02:48:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742420_1596 (size=12459) 2024-11-24T02:48:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742421_1597 (size=9814) 2024-11-24T02:48:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,818 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112434fad882d07a455aa543dc45d53c77c5_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112434fad882d07a455aa543dc45d53c77c5_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5c1aaa6b864646e5a822e9eee3ae34f4, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5c1aaa6b864646e5a822e9eee3ae34f4 is 175, key is test_row_0/A:col10/1732416533385/Put/seqid=0 2024-11-24T02:48:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742422_1598 (size=22461) 2024-11-24T02:48:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,161 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#A#compaction#503 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:55,162 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/8e19cb74c1c34345b03674ab373237c2 is 175, key is test_row_0/A:col10/1732416531207/Put/seqid=0 2024-11-24T02:48:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742423_1599 (size=31413) 2024-11-24T02:48:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,214 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/3a6befabe0294df59f48ce6677931db5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/3a6befabe0294df59f48ce6677931db5 2024-11-24T02:48:55,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,217 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/C of 824d61e10dc42fc1ca4048b972dd9659 into 3a6befabe0294df59f48ce6677931db5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:55,217 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:55,218 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/C, priority=12, startTime=1732416534679; duration=0sec 2024-11-24T02:48:55,218 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:55,218 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:C 2024-11-24T02:48:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,230 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5c1aaa6b864646e5a822e9eee3ae34f4 2024-11-24T02:48:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/41bedafd8cc1439eb332df3ed39ea91e is 50, key is test_row_0/B:col10/1732416533385/Put/seqid=0 2024-11-24T02:48:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742424_1600 (size=9757) 2024-11-24T02:48:55,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,252 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/41bedafd8cc1439eb332df3ed39ea91e 2024-11-24T02:48:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/601af336068c4287854dd3e01e1910c9 is 50, key is test_row_0/C:col10/1732416533385/Put/seqid=0 2024-11-24T02:48:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742425_1601 (size=9757) 2024-11-24T02:48:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:55,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T02:48:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416595412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416595416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416595416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416595417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,422 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416595526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416595526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416595527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416595527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416595566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,569 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:55,570 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/8e19cb74c1c34345b03674ab373237c2 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/8e19cb74c1c34345b03674ab373237c2 2024-11-24T02:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,573 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/A of 824d61e10dc42fc1ca4048b972dd9659 into 8e19cb74c1c34345b03674ab373237c2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:55,573 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:55,573 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/A, priority=12, startTime=1732416534678; duration=0sec 2024-11-24T02:48:55,574 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:55,574 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:A 2024-11-24T02:48:55,667 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/601af336068c4287854dd3e01e1910c9 2024-11-24T02:48:55,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5c1aaa6b864646e5a822e9eee3ae34f4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5c1aaa6b864646e5a822e9eee3ae34f4 2024-11-24T02:48:55,674 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5c1aaa6b864646e5a822e9eee3ae34f4, entries=100, sequenceid=165, filesize=21.9 K 2024-11-24T02:48:55,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/41bedafd8cc1439eb332df3ed39ea91e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/41bedafd8cc1439eb332df3ed39ea91e 2024-11-24T02:48:55,679 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/41bedafd8cc1439eb332df3ed39ea91e, entries=100, sequenceid=165, filesize=9.5 K 2024-11-24T02:48:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/601af336068c4287854dd3e01e1910c9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/601af336068c4287854dd3e01e1910c9 2024-11-24T02:48:55,682 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/601af336068c4287854dd3e01e1910c9, entries=100, sequenceid=165, filesize=9.5 K 2024-11-24T02:48:55,683 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 824d61e10dc42fc1ca4048b972dd9659 in 898ms, sequenceid=165, compaction requested=false 2024-11-24T02:48:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-24T02:48:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-24T02:48:55,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-24T02:48:55,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4340 sec 2024-11-24T02:48:55,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.4390 sec 2024-11-24T02:48:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:55,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:55,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:48:55,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416595743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416595746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243cf5cf63bb5746fca314cf3b00826a9b_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416535416/Put/seqid=0 2024-11-24T02:48:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416595748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416595748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742426_1602 (size=12304) 2024-11-24T02:48:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416595856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416595856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416595856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416595857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416596066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416596066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416596066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416596066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,162 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:56,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243cf5cf63bb5746fca314cf3b00826a9b_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243cf5cf63bb5746fca314cf3b00826a9b_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:56,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5ab650af56eb4979b228b3d6336b1d95, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:56,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5ab650af56eb4979b228b3d6336b1d95 is 175, key is test_row_0/A:col10/1732416535416/Put/seqid=0 2024-11-24T02:48:56,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742427_1603 (size=31105) 2024-11-24T02:48:56,169 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5ab650af56eb4979b228b3d6336b1d95 2024-11-24T02:48:56,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/271f0e10b93145bc919bf06e09528463 is 50, key is test_row_0/B:col10/1732416535416/Put/seqid=0 2024-11-24T02:48:56,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742428_1604 (size=12151) 2024-11-24T02:48:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416596377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416596377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416596377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416596377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/271f0e10b93145bc919bf06e09528463 2024-11-24T02:48:56,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/d7a0e5f8c5034cb9bacfefcd6d552489 is 50, key is test_row_0/C:col10/1732416535416/Put/seqid=0 2024-11-24T02:48:56,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742429_1605 (size=12151) 2024-11-24T02:48:56,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416596886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416596887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416596887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416596887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:56,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/d7a0e5f8c5034cb9bacfefcd6d552489 2024-11-24T02:48:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/5ab650af56eb4979b228b3d6336b1d95 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5ab650af56eb4979b228b3d6336b1d95 2024-11-24T02:48:56,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5ab650af56eb4979b228b3d6336b1d95, entries=150, sequenceid=194, filesize=30.4 K 2024-11-24T02:48:56,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/271f0e10b93145bc919bf06e09528463 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/271f0e10b93145bc919bf06e09528463 2024-11-24T02:48:56,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/271f0e10b93145bc919bf06e09528463, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T02:48:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/d7a0e5f8c5034cb9bacfefcd6d552489 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/d7a0e5f8c5034cb9bacfefcd6d552489 2024-11-24T02:48:56,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/d7a0e5f8c5034cb9bacfefcd6d552489, entries=150, sequenceid=194, filesize=11.9 K 2024-11-24T02:48:56,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 824d61e10dc42fc1ca4048b972dd9659 in 1259ms, sequenceid=194, compaction requested=true 2024-11-24T02:48:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:48:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:56,999 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:48:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:56,999 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:48:56,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/B is initiating minor compaction (all files) 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/A is initiating minor compaction (all files) 2024-11-24T02:48:57,000 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/B in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:57,000 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/A in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:57,000 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/8e19cb74c1c34345b03674ab373237c2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5c1aaa6b864646e5a822e9eee3ae34f4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5ab650af56eb4979b228b3d6336b1d95] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=83.0 K 2024-11-24T02:48:57,000 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/b3d9cda38f86415b8c73abc80b319719, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/41bedafd8cc1439eb332df3ed39ea91e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/271f0e10b93145bc919bf06e09528463] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=33.6 K 2024-11-24T02:48:57,000 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/8e19cb74c1c34345b03674ab373237c2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5c1aaa6b864646e5a822e9eee3ae34f4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5ab650af56eb4979b228b3d6336b1d95] 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d9cda38f86415b8c73abc80b319719, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416531187 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e19cb74c1c34345b03674ab373237c2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416531187 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1aaa6b864646e5a822e9eee3ae34f4, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732416533385 2024-11-24T02:48:57,000 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 41bedafd8cc1439eb332df3ed39ea91e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732416533385 2024-11-24T02:48:57,001 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ab650af56eb4979b228b3d6336b1d95, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732416535414 2024-11-24T02:48:57,001 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 271f0e10b93145bc919bf06e09528463, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732416535414 2024-11-24T02:48:57,007 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#B#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:57,007 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/3dc841662e034adabb82777348d95fac is 50, key is test_row_0/B:col10/1732416535416/Put/seqid=0 2024-11-24T02:48:57,009 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:57,028 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124a0a8979254d2427b96db782cde70b5d2_824d61e10dc42fc1ca4048b972dd9659 store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:57,029 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124a0a8979254d2427b96db782cde70b5d2_824d61e10dc42fc1ca4048b972dd9659, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:57,029 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a0a8979254d2427b96db782cde70b5d2_824d61e10dc42fc1ca4048b972dd9659 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:57,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742430_1606 (size=12561) 2024-11-24T02:48:57,036 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/3dc841662e034adabb82777348d95fac as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3dc841662e034adabb82777348d95fac 2024-11-24T02:48:57,042 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/B of 824d61e10dc42fc1ca4048b972dd9659 into 3dc841662e034adabb82777348d95fac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:57,042 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:57,042 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/B, priority=13, startTime=1732416536999; duration=0sec 2024-11-24T02:48:57,043 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:48:57,043 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:B 2024-11-24T02:48:57,043 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:48:57,046 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:48:57,046 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/C is initiating minor compaction (all files) 2024-11-24T02:48:57,046 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/C in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:57,046 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/3a6befabe0294df59f48ce6677931db5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/601af336068c4287854dd3e01e1910c9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/d7a0e5f8c5034cb9bacfefcd6d552489] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=33.6 K 2024-11-24T02:48:57,047 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a6befabe0294df59f48ce6677931db5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732416531187 2024-11-24T02:48:57,047 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 601af336068c4287854dd3e01e1910c9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732416533385 2024-11-24T02:48:57,047 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting d7a0e5f8c5034cb9bacfefcd6d552489, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732416535414 2024-11-24T02:48:57,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742431_1607 (size=4469) 2024-11-24T02:48:57,063 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#C#compaction#514 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:57,063 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#A#compaction#513 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:48:57,063 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/1388e60844064fde8c6c02d3533d883c is 50, key is test_row_0/C:col10/1732416535416/Put/seqid=0 2024-11-24T02:48:57,064 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/9fe4615aced04299a1d7d195b31922a4 is 175, key is test_row_0/A:col10/1732416535416/Put/seqid=0 2024-11-24T02:48:57,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742432_1608 (size=12561) 2024-11-24T02:48:57,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742433_1609 (size=31515) 2024-11-24T02:48:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-24T02:48:57,417 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:57,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-24T02:48:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:48:57,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:57,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:57,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:57,481 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/9fe4615aced04299a1d7d195b31922a4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9fe4615aced04299a1d7d195b31922a4 2024-11-24T02:48:57,481 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/1388e60844064fde8c6c02d3533d883c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/1388e60844064fde8c6c02d3533d883c 2024-11-24T02:48:57,485 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/C of 824d61e10dc42fc1ca4048b972dd9659 into 1388e60844064fde8c6c02d3533d883c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:57,485 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/A of 824d61e10dc42fc1ca4048b972dd9659 into 9fe4615aced04299a1d7d195b31922a4(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:48:57,485 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:57,485 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:57,485 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/C, priority=13, startTime=1732416536999; duration=0sec 2024-11-24T02:48:57,485 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/A, priority=13, startTime=1732416536999; duration=0sec 2024-11-24T02:48:57,485 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:57,485 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:48:57,485 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:A 2024-11-24T02:48:57,485 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:C 2024-11-24T02:48:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:48:57,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-24T02:48:57,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:57,572 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-24T02:48:57,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124286f60b1ca954d7f9e5bbe3a0350f5c8_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416535741/Put/seqid=0 2024-11-24T02:48:57,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742434_1610 (size=12304) 2024-11-24T02:48:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:57,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:48:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416597682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:48:57,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416597796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416597896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416597896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416597897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:57,899 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416597897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:57,899 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,899 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_0' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:57,900 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 7c69a60bd8f6,39461,1732416369827 for 'test_row_1' in 824d61e10dc42fc1ca4048b972dd9659 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:57,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:57,988 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124286f60b1ca954d7f9e5bbe3a0350f5c8_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124286f60b1ca954d7f9e5bbe3a0350f5c8_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:57,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/0d2b8cf86a31439995c9e44393a1de6d, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:57,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/0d2b8cf86a31439995c9e44393a1de6d is 175, key is test_row_0/A:col10/1732416535741/Put/seqid=0 2024-11-24T02:48:57,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742435_1611 (size=31105) 2024-11-24T02:48:57,993 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/0d2b8cf86a31439995c9e44393a1de6d 2024-11-24T02:48:57,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/3e006a87b37647c398ca03b445e2545d is 50, key is test_row_0/B:col10/1732416535741/Put/seqid=0 2024-11-24T02:48:58,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742436_1612 (size=12151) 2024-11-24T02:48:58,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416598007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:58,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:48:58,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416598316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:58,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:58,403 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/3e006a87b37647c398ca03b445e2545d 2024-11-24T02:48:58,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/e5b1d01bc2ae4167bed9de8306c3029a is 50, key is test_row_0/C:col10/1732416535741/Put/seqid=0 2024-11-24T02:48:58,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742437_1613 (size=12151) 2024-11-24T02:48:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:48:58,820 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/e5b1d01bc2ae4167bed9de8306c3029a 2024-11-24T02:48:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/0d2b8cf86a31439995c9e44393a1de6d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0d2b8cf86a31439995c9e44393a1de6d 2024-11-24T02:48:58,826 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0d2b8cf86a31439995c9e44393a1de6d, entries=150, sequenceid=206, filesize=30.4 K 2024-11-24T02:48:58,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/3e006a87b37647c398ca03b445e2545d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3e006a87b37647c398ca03b445e2545d 2024-11-24T02:48:58,830 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3e006a87b37647c398ca03b445e2545d, entries=150, sequenceid=206, filesize=11.9 K 2024-11-24T02:48:58,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416598827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:58,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:58,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/e5b1d01bc2ae4167bed9de8306c3029a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e5b1d01bc2ae4167bed9de8306c3029a 2024-11-24T02:48:58,836 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e5b1d01bc2ae4167bed9de8306c3029a, entries=150, sequenceid=206, filesize=11.9 K 2024-11-24T02:48:58,837 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 824d61e10dc42fc1ca4048b972dd9659 in 1265ms, sequenceid=206, compaction requested=false 2024-11-24T02:48:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:58,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-24T02:48:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-24T02:48:58,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-24T02:48:58,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-11-24T02:48:58,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.4230 sec 2024-11-24T02:48:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T02:48:59,566 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:59,567 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-24T02:48:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T02:48:59,568 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:59,569 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:59,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T02:48:59,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-24T02:48:59,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:59,722 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-24T02:48:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245b74465eb04b47328164ebd547d08165_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416537681/Put/seqid=0 2024-11-24T02:48:59,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742438_1614 (size=12304) 2024-11-24T02:48:59,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,747 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245b74465eb04b47328164ebd547d08165_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245b74465eb04b47328164ebd547d08165_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:59,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/04b3f8cf6ff94bd3ac41d5cfa329aaba, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/04b3f8cf6ff94bd3ac41d5cfa329aaba is 175, key is test_row_0/A:col10/1732416537681/Put/seqid=0 2024-11-24T02:48:59,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742439_1615 (size=31105) 2024-11-24T02:48:59,755 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/04b3f8cf6ff94bd3ac41d5cfa329aaba 2024-11-24T02:48:59,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/67ee5b66638b4e428ff65bab227134d4 is 50, key is test_row_0/B:col10/1732416537681/Put/seqid=0 2024-11-24T02:48:59,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742440_1616 (size=12151) 2024-11-24T02:48:59,785 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/67ee5b66638b4e428ff65bab227134d4 2024-11-24T02:48:59,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/f02cf4f581044bec8be5e415d96d7c2c is 50, key is test_row_0/C:col10/1732416537681/Put/seqid=0 2024-11-24T02:48:59,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742441_1617 (size=12151) 2024-11-24T02:48:59,800 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/f02cf4f581044bec8be5e415d96d7c2c 2024-11-24T02:48:59,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/04b3f8cf6ff94bd3ac41d5cfa329aaba as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/04b3f8cf6ff94bd3ac41d5cfa329aaba 2024-11-24T02:48:59,813 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/04b3f8cf6ff94bd3ac41d5cfa329aaba, entries=150, sequenceid=233, filesize=30.4 K 2024-11-24T02:48:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/67ee5b66638b4e428ff65bab227134d4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/67ee5b66638b4e428ff65bab227134d4 2024-11-24T02:48:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,819 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/67ee5b66638b4e428ff65bab227134d4, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T02:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/f02cf4f581044bec8be5e415d96d7c2c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/f02cf4f581044bec8be5e415d96d7c2c 2024-11-24T02:48:59,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/f02cf4f581044bec8be5e415d96d7c2c, entries=150, sequenceid=233, filesize=11.9 K 2024-11-24T02:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,824 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=0 B/0 for 824d61e10dc42fc1ca4048b972dd9659 in 102ms, sequenceid=233, compaction requested=true 2024-11-24T02:48:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:48:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:48:59,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-24T02:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-24T02:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-24T02:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 257 msec 2024-11-24T02:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 261 msec 2024-11-24T02:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-24T02:48:59,886 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:48:59,887 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:48:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-24T02:48:59,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,889 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:48:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T02:48:59,890 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:48:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:48:59,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:59,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:48:59,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-24T02:48:59,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242fcb19e0af5f458285c6f056485ba1cd_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416539924/Put/seqid=0 2024-11-24T02:48:59,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742442_1618 (size=14794) 2024-11-24T02:48:59,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:48:59,950 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242fcb19e0af5f458285c6f056485ba1cd_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242fcb19e0af5f458285c6f056485ba1cd_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:48:59,951 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/814e3100b4854bafba72cc7cc4ea4c3a, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:48:59,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/814e3100b4854bafba72cc7cc4ea4c3a is 175, key is test_row_0/A:col10/1732416539924/Put/seqid=0 2024-11-24T02:48:59,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742443_1619 (size=39749) 2024-11-24T02:48:59,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416599950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:59,957 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/814e3100b4854bafba72cc7cc4ea4c3a 2024-11-24T02:48:59,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416599954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416599957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416599957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:48:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:59,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416599958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:48:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:48:59,964 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:48:59,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/6807a225df014467928c2960ab77766d is 50, key is test_row_0/B:col10/1732416539924/Put/seqid=0 2024-11-24T02:48:59,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742444_1620 (size=12151) 2024-11-24T02:48:59,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/6807a225df014467928c2960ab77766d 2024-11-24T02:48:59,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/c46ade6861c04b2996dc6894d8debc66 is 50, key is test_row_0/C:col10/1732416539924/Put/seqid=0 2024-11-24T02:48:59,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742445_1621 (size=12151) 2024-11-24T02:48:59,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/c46ade6861c04b2996dc6894d8debc66 2024-11-24T02:48:59,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/814e3100b4854bafba72cc7cc4ea4c3a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/814e3100b4854bafba72cc7cc4ea4c3a 2024-11-24T02:48:59,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/814e3100b4854bafba72cc7cc4ea4c3a, entries=200, sequenceid=245, filesize=38.8 K 2024-11-24T02:48:59,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/6807a225df014467928c2960ab77766d as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/6807a225df014467928c2960ab77766d 2024-11-24T02:48:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T02:49:00,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/6807a225df014467928c2960ab77766d, entries=150, sequenceid=245, filesize=11.9 K 2024-11-24T02:49:00,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/c46ade6861c04b2996dc6894d8debc66 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c46ade6861c04b2996dc6894d8debc66 2024-11-24T02:49:00,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c46ade6861c04b2996dc6894d8debc66, entries=150, sequenceid=245, filesize=11.9 K 2024-11-24T02:49:00,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 824d61e10dc42fc1ca4048b972dd9659 in 82ms, sequenceid=245, compaction requested=true 2024-11-24T02:49:00,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:00,006 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:49:00,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:49:00,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:00,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:49:00,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:00,007 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:49:00,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:49:00,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:49:00,009 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:49:00,009 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133474 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:49:00,009 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/B is initiating minor compaction (all files) 2024-11-24T02:49:00,009 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/A is initiating minor compaction (all files) 2024-11-24T02:49:00,010 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/A in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:00,010 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/B in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:00,010 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3dc841662e034adabb82777348d95fac, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3e006a87b37647c398ca03b445e2545d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/67ee5b66638b4e428ff65bab227134d4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/6807a225df014467928c2960ab77766d] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=47.9 K 2024-11-24T02:49:00,010 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9fe4615aced04299a1d7d195b31922a4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0d2b8cf86a31439995c9e44393a1de6d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/04b3f8cf6ff94bd3ac41d5cfa329aaba, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/814e3100b4854bafba72cc7cc4ea4c3a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=130.3 K 2024-11-24T02:49:00,010 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:00,010 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9fe4615aced04299a1d7d195b31922a4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0d2b8cf86a31439995c9e44393a1de6d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/04b3f8cf6ff94bd3ac41d5cfa329aaba, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/814e3100b4854bafba72cc7cc4ea4c3a] 2024-11-24T02:49:00,010 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dc841662e034adabb82777348d95fac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732416535414 2024-11-24T02:49:00,010 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fe4615aced04299a1d7d195b31922a4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732416535414 2024-11-24T02:49:00,010 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e006a87b37647c398ca03b445e2545d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732416535741 2024-11-24T02:49:00,010 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d2b8cf86a31439995c9e44393a1de6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732416535741 2024-11-24T02:49:00,011 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 67ee5b66638b4e428ff65bab227134d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732416537665 2024-11-24T02:49:00,011 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b3f8cf6ff94bd3ac41d5cfa329aaba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732416537665 2024-11-24T02:49:00,011 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6807a225df014467928c2960ab77766d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732416539888 2024-11-24T02:49:00,011 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 814e3100b4854bafba72cc7cc4ea4c3a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732416539888 2024-11-24T02:49:00,019 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:00,027 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:49:00,027 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/74fbccedf6eb4719918ff400d914d7d8 is 50, key is test_row_0/B:col10/1732416539924/Put/seqid=0 2024-11-24T02:49:00,033 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411240be44b9fcf88475b894c3be37440c22f_824d61e10dc42fc1ca4048b972dd9659 store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:00,038 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411240be44b9fcf88475b894c3be37440c22f_824d61e10dc42fc1ca4048b972dd9659, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:00,038 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240be44b9fcf88475b894c3be37440c22f_824d61e10dc42fc1ca4048b972dd9659 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:00,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-24T02:49:00,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:00,043 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-24T02:49:00,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742446_1622 (size=12697) 2024-11-24T02:49:00,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742447_1623 (size=4469) 2024-11-24T02:49:00,067 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#A#compaction#524 average throughput is 0.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:49:00,067 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/b8f7cf857b64492bb08d03ba297b3652 is 175, key is test_row_0/A:col10/1732416539924/Put/seqid=0 2024-11-24T02:49:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:00,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:49:00,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ea23365559f642bfb1c46ac117f2d270_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416539954/Put/seqid=0 2024-11-24T02:49:00,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416600075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416600082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416600082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416600083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416600083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742448_1624 (size=31651) 2024-11-24T02:49:00,112 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/b8f7cf857b64492bb08d03ba297b3652 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/b8f7cf857b64492bb08d03ba297b3652 2024-11-24T02:49:00,118 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/A of 824d61e10dc42fc1ca4048b972dd9659 into b8f7cf857b64492bb08d03ba297b3652(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:49:00,118 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:00,118 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/A, priority=12, startTime=1732416540006; duration=0sec 2024-11-24T02:49:00,118 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:49:00,118 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:A 2024-11-24T02:49:00,118 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T02:49:00,119 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T02:49:00,119 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/C is initiating minor compaction (all files) 2024-11-24T02:49:00,119 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/C in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:00,119 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/1388e60844064fde8c6c02d3533d883c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e5b1d01bc2ae4167bed9de8306c3029a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/f02cf4f581044bec8be5e415d96d7c2c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c46ade6861c04b2996dc6894d8debc66] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=47.9 K 2024-11-24T02:49:00,119 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1388e60844064fde8c6c02d3533d883c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732416535414 2024-11-24T02:49:00,120 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5b1d01bc2ae4167bed9de8306c3029a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732416535741 2024-11-24T02:49:00,120 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02cf4f581044bec8be5e415d96d7c2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732416537665 2024-11-24T02:49:00,120 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c46ade6861c04b2996dc6894d8debc66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732416539888 2024-11-24T02:49:00,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742449_1625 (size=12454) 2024-11-24T02:49:00,136 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#C#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:49:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:49:00,137 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/dbb34ad6fc9648e78e842ce973cf3b8f is 50, key is test_row_0/C:col10/1732416539924/Put/seqid=0 2024-11-24T02:49:00,141 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124ea23365559f642bfb1c46ac117f2d270_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ea23365559f642bfb1c46ac117f2d270_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:00,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742450_1626 (size=12697) 2024-11-24T02:49:00,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/1727d4693e574b0c9227b7770e306a99, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:00,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/1727d4693e574b0c9227b7770e306a99 is 175, key is test_row_0/A:col10/1732416539954/Put/seqid=0 2024-11-24T02:49:00,155 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/dbb34ad6fc9648e78e842ce973cf3b8f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/dbb34ad6fc9648e78e842ce973cf3b8f 2024-11-24T02:49:00,161 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/C of 824d61e10dc42fc1ca4048b972dd9659 into dbb34ad6fc9648e78e842ce973cf3b8f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:49:00,161 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:00,161 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/C, priority=12, startTime=1732416540007; duration=0sec 2024-11-24T02:49:00,161 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:00,161 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:C 2024-11-24T02:49:00,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742451_1627 (size=31255) 2024-11-24T02:49:00,167 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/1727d4693e574b0c9227b7770e306a99 2024-11-24T02:49:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/8b78c34da51f4f2198cfaac9de35fcbf is 50, key is test_row_0/B:col10/1732416539954/Put/seqid=0 2024-11-24T02:49:00,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742452_1628 (size=12301) 2024-11-24T02:49:00,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416600187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416600196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416600196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416600196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416600196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T02:49:00,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416600400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416600406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416600407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416600407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416600407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,461 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/74fbccedf6eb4719918ff400d914d7d8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/74fbccedf6eb4719918ff400d914d7d8 2024-11-24T02:49:00,465 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/B of 824d61e10dc42fc1ca4048b972dd9659 into 74fbccedf6eb4719918ff400d914d7d8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:49:00,465 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:00,465 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/B, priority=12, startTime=1732416540007; duration=0sec 2024-11-24T02:49:00,465 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:00,465 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:B 2024-11-24T02:49:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T02:49:00,588 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/8b78c34da51f4f2198cfaac9de35fcbf 2024-11-24T02:49:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/4e93a9d4d33c45ff832206d8d76ff9c9 is 50, key is test_row_0/C:col10/1732416539954/Put/seqid=0 2024-11-24T02:49:00,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742453_1629 (size=12301) 2024-11-24T02:49:00,645 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/4e93a9d4d33c45ff832206d8d76ff9c9 2024-11-24T02:49:00,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/1727d4693e574b0c9227b7770e306a99 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/1727d4693e574b0c9227b7770e306a99 2024-11-24T02:49:00,657 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/1727d4693e574b0c9227b7770e306a99, entries=150, sequenceid=270, filesize=30.5 K 2024-11-24T02:49:00,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/8b78c34da51f4f2198cfaac9de35fcbf as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/8b78c34da51f4f2198cfaac9de35fcbf 2024-11-24T02:49:00,663 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/8b78c34da51f4f2198cfaac9de35fcbf, entries=150, sequenceid=270, filesize=12.0 K 2024-11-24T02:49:00,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/4e93a9d4d33c45ff832206d8d76ff9c9 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/4e93a9d4d33c45ff832206d8d76ff9c9 2024-11-24T02:49:00,669 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/4e93a9d4d33c45ff832206d8d76ff9c9, entries=150, sequenceid=270, filesize=12.0 K 2024-11-24T02:49:00,670 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 824d61e10dc42fc1ca4048b972dd9659 in 627ms, sequenceid=270, compaction requested=false 2024-11-24T02:49:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-24T02:49:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-24T02:49:00,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-24T02:49:00,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 782 msec 2024-11-24T02:49:00,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 788 msec 2024-11-24T02:49:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:00,709 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:49:00,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-24T02:49:00,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f7779e8581d04b01b6f1d0b6a9a73eaf_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416540081/Put/seqid=0 2024-11-24T02:49:00,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742454_1630 (size=14994) 2024-11-24T02:49:00,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416600742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416600746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416600774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416600775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416600775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416600877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416600877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416600886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416600886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416600886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:00,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-24T02:49:01,027 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:49:01,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T02:49:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-24T02:49:01,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T02:49:01,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T02:49:01,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T02:49:01,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T02:49:01,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416601086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416601086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416601096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416601096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416601096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T02:49:01,147 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:49:01,150 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f7779e8581d04b01b6f1d0b6a9a73eaf_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f7779e8581d04b01b6f1d0b6a9a73eaf_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:01,151 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/80fb9933922b45fa87605683c8bfea8c, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:01,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/80fb9933922b45fa87605683c8bfea8c is 175, key is test_row_0/A:col10/1732416540081/Put/seqid=0 2024-11-24T02:49:01,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742455_1631 (size=39949) 2024-11-24T02:49:01,155 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/80fb9933922b45fa87605683c8bfea8c 2024-11-24T02:49:01,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/910a83ae1d38438da385e45bba5004d5 is 50, key is test_row_0/B:col10/1732416540081/Put/seqid=0 2024-11-24T02:49:01,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742456_1632 (size=12301) 2024-11-24T02:49:01,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-24T02:49:01,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:49:01,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-24T02:49:01,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:49:01,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T02:49:01,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416601397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416601397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416601406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416601406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416601406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-24T02:49:01,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:49:01,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:49:01,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/910a83ae1d38438da385e45bba5004d5 2024-11-24T02:49:01,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/98a068962e7045ddb6393815bf13a43a is 50, key is test_row_0/C:col10/1732416540081/Put/seqid=0 2024-11-24T02:49:01,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742457_1633 (size=12301) 2024-11-24T02:49:01,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/98a068962e7045ddb6393815bf13a43a 2024-11-24T02:49:01,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/80fb9933922b45fa87605683c8bfea8c as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/80fb9933922b45fa87605683c8bfea8c 2024-11-24T02:49:01,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/80fb9933922b45fa87605683c8bfea8c, entries=200, sequenceid=286, filesize=39.0 K 2024-11-24T02:49:01,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/910a83ae1d38438da385e45bba5004d5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/910a83ae1d38438da385e45bba5004d5 2024-11-24T02:49:01,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/910a83ae1d38438da385e45bba5004d5, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T02:49:01,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/98a068962e7045ddb6393815bf13a43a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/98a068962e7045ddb6393815bf13a43a 2024-11-24T02:49:01,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/98a068962e7045ddb6393815bf13a43a, entries=150, sequenceid=286, filesize=12.0 K 2024-11-24T02:49:01,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 824d61e10dc42fc1ca4048b972dd9659 in 880ms, sequenceid=286, compaction requested=true 2024-11-24T02:49:01,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:01,590 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:49:01,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:49:01,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:01,590 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:49:01,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:49:01,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:01,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T02:49:01,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:49:01,590 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:49:01,590 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/A is initiating minor compaction (all files) 2024-11-24T02:49:01,590 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:49:01,590 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/A in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,591 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/B is initiating minor compaction (all files) 2024-11-24T02:49:01,591 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/b8f7cf857b64492bb08d03ba297b3652, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/1727d4693e574b0c9227b7770e306a99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/80fb9933922b45fa87605683c8bfea8c] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=100.4 K 2024-11-24T02:49:01,591 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/B in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,591 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,591 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/74fbccedf6eb4719918ff400d914d7d8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/8b78c34da51f4f2198cfaac9de35fcbf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/910a83ae1d38438da385e45bba5004d5] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=36.4 K 2024-11-24T02:49:01,591 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. files: [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/b8f7cf857b64492bb08d03ba297b3652, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/1727d4693e574b0c9227b7770e306a99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/80fb9933922b45fa87605683c8bfea8c] 2024-11-24T02:49:01,591 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8f7cf857b64492bb08d03ba297b3652, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732416539888 2024-11-24T02:49:01,591 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 74fbccedf6eb4719918ff400d914d7d8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732416539888 2024-11-24T02:49:01,591 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b78c34da51f4f2198cfaac9de35fcbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732416539940 2024-11-24T02:49:01,591 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1727d4693e574b0c9227b7770e306a99, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732416539940 2024-11-24T02:49:01,592 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] compactions.Compactor(225): Compacting 910a83ae1d38438da385e45bba5004d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732416540081 2024-11-24T02:49:01,592 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80fb9933922b45fa87605683c8bfea8c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732416540070 2024-11-24T02:49:01,602 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:01,603 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#B#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:49:01,604 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/61d45a070e524ce48d7fa9566d3e8e21 is 50, key is test_row_0/B:col10/1732416540081/Put/seqid=0 2024-11-24T02:49:01,604 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124931f4c86485a4dd8ae225cd41ab7e280_824d61e10dc42fc1ca4048b972dd9659 store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:01,606 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124931f4c86485a4dd8ae225cd41ab7e280_824d61e10dc42fc1ca4048b972dd9659, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:01,606 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124931f4c86485a4dd8ae225cd41ab7e280_824d61e10dc42fc1ca4048b972dd9659 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:01,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742458_1634 (size=12949) 2024-11-24T02:49:01,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742459_1635 (size=4469) 2024-11-24T02:49:01,623 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#A#compaction#534 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:49:01,624 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/0110e087e88d43baa4a4c728c9fc910f is 175, key is test_row_0/A:col10/1732416540081/Put/seqid=0 2024-11-24T02:49:01,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742460_1636 (size=31903) 2024-11-24T02:49:01,632 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/0110e087e88d43baa4a4c728c9fc910f as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0110e087e88d43baa4a4c728c9fc910f 2024-11-24T02:49:01,636 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/A of 824d61e10dc42fc1ca4048b972dd9659 into 0110e087e88d43baa4a4c728c9fc910f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:49:01,636 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:01,636 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/A, priority=13, startTime=1732416541589; duration=0sec 2024-11-24T02:49:01,636 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T02:49:01,636 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:A 2024-11-24T02:49:01,636 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T02:49:01,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T02:49:01,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1541): 824d61e10dc42fc1ca4048b972dd9659/C is initiating minor compaction (all files) 2024-11-24T02:49:01,637 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 824d61e10dc42fc1ca4048b972dd9659/C in TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,637 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/dbb34ad6fc9648e78e842ce973cf3b8f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/4e93a9d4d33c45ff832206d8d76ff9c9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/98a068962e7045ddb6393815bf13a43a] into tmpdir=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp, totalSize=36.4 K 2024-11-24T02:49:01,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbb34ad6fc9648e78e842ce973cf3b8f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732416539888 2024-11-24T02:49:01,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e93a9d4d33c45ff832206d8d76ff9c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732416539940 2024-11-24T02:49:01,637 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a068962e7045ddb6393815bf13a43a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732416540081 2024-11-24T02:49:01,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-24T02:49:01,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:01,639 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-24T02:49:01,643 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 824d61e10dc42fc1ca4048b972dd9659#C#compaction#535 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T02:49:01,644 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/727c9ab9224540129cab23e42a3ae1f3 is 50, key is test_row_0/C:col10/1732416540081/Put/seqid=0 2024-11-24T02:49:01,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2fa877cc1d142c980608089a7ec036a_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416540730/Put/seqid=0 2024-11-24T02:49:01,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742461_1637 (size=12949) 2024-11-24T02:49:01,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742462_1638 (size=12454) 2024-11-24T02:49:01,651 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/727c9ab9224540129cab23e42a3ae1f3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/727c9ab9224540129cab23e42a3ae1f3 2024-11-24T02:49:01,654 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/C of 824d61e10dc42fc1ca4048b972dd9659 into 727c9ab9224540129cab23e42a3ae1f3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:49:01,654 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:01,654 INFO [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/C, priority=13, startTime=1732416541590; duration=0sec 2024-11-24T02:49:01,654 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:01,654 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:C 2024-11-24T02:49:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T02:49:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:01,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. as already flushing 2024-11-24T02:49:01,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416601918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416601918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416601919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416601920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416601921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:01,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,026 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/61d45a070e524ce48d7fa9566d3e8e21 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/61d45a070e524ce48d7fa9566d3e8e21 2024-11-24T02:49:02,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416602027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416602027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416602027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416602028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,030 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 824d61e10dc42fc1ca4048b972dd9659/B of 824d61e10dc42fc1ca4048b972dd9659 into 61d45a070e524ce48d7fa9566d3e8e21(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T02:49:02,030 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:02,030 INFO [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., storeName=824d61e10dc42fc1ca4048b972dd9659/B, priority=13, startTime=1732416541590; duration=0sec 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416602028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,030 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:02,030 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:B 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:49:02,052 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c2fa877cc1d142c980608089a7ec036a_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2fa877cc1d142c980608089a7ec036a_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:02,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/7c4e4cf6b7c74814ad2698f0e19684c6, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:02,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/7c4e4cf6b7c74814ad2698f0e19684c6 is 175, key is test_row_0/A:col10/1732416540730/Put/seqid=0 2024-11-24T02:49:02,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742463_1639 (size=31255) 2024-11-24T02:49:02,056 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/7c4e4cf6b7c74814ad2698f0e19684c6 2024-11-24T02:49:02,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/88d72de7cc2c450d9c095f24a028eb85 is 50, key is test_row_0/B:col10/1732416540730/Put/seqid=0 2024-11-24T02:49:02,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742464_1640 (size=12301) 2024-11-24T02:49:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T02:49:02,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416602237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416602237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416602237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416602237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416602237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,469 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/88d72de7cc2c450d9c095f24a028eb85 2024-11-24T02:49:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/e4c2c823a346408bb3de0e3b533f170a is 50, key is test_row_0/C:col10/1732416540730/Put/seqid=0 2024-11-24T02:49:02,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742465_1641 (size=12301) 2024-11-24T02:49:02,536 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/e4c2c823a346408bb3de0e3b533f170a 2024-11-24T02:49:02,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/7c4e4cf6b7c74814ad2698f0e19684c6 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/7c4e4cf6b7c74814ad2698f0e19684c6 2024-11-24T02:49:02,542 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/7c4e4cf6b7c74814ad2698f0e19684c6, entries=150, sequenceid=311, filesize=30.5 K 2024-11-24T02:49:02,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/88d72de7cc2c450d9c095f24a028eb85 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/88d72de7cc2c450d9c095f24a028eb85 2024-11-24T02:49:02,545 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/88d72de7cc2c450d9c095f24a028eb85, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T02:49:02,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/e4c2c823a346408bb3de0e3b533f170a as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e4c2c823a346408bb3de0e3b533f170a 2024-11-24T02:49:02,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56716 deadline: 1732416602547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56752 deadline: 1732416602547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56714 deadline: 1732416602547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56764 deadline: 1732416602547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56690 deadline: 1732416602547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 , the old value is region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T02:49:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659., hostname=7c69a60bd8f6,39461,1732416369827, seqNum=5 because the exception is null or not the one we care about 2024-11-24T02:49:02,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e4c2c823a346408bb3de0e3b533f170a, entries=150, sequenceid=311, filesize=12.0 K 2024-11-24T02:49:02,551 INFO [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 824d61e10dc42fc1ca4048b972dd9659 in 912ms, sequenceid=311, compaction requested=false 2024-11-24T02:49:02,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:02,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:02,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c69a60bd8f6:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-24T02:49:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-24T02:49:02,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-24T02:49:02,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5230 sec 2024-11-24T02:49:02,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 1.5270 sec 2024-11-24T02:49:02,632 INFO [Thread-2492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2492. 2024-11-24T02:49:02,632 INFO [Thread-2484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2484. 2024-11-24T02:49:02,632 DEBUG [Thread-2484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:02,632 DEBUG [Thread-2492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:02,632 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,632 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,632 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,632 DEBUG [Thread-2484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:02,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:02,632 INFO [Thread-2490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2490. 2024-11-24T02:49:02,632 DEBUG [Thread-2490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:02,632 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,632 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:02,633 INFO [Thread-2488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2488. 2024-11-24T02:49:02,633 DEBUG [Thread-2488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:02,633 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,633 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,633 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:02,634 INFO [Thread-2486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2486. 2024-11-24T02:49:02,634 DEBUG [Thread-2486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:02,634 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,634 DEBUG [Thread-2486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:02,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39461 {}] regionserver.HRegion(8855): Flush requested on 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-24T02:49:03,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 824d61e10dc42fc1ca4048b972dd9659 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-24T02:49:03,058 INFO [Thread-2481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2481. 2024-11-24T02:49:03,058 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-24T02:49:03,058 INFO [Thread-2477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2477. 2024-11-24T02:49:03,058 DEBUG [Thread-2481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:03,058 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:03,058 DEBUG [Thread-2477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:03,058 DEBUG [Thread-2481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,058 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,058 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,058 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,058 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,058 DEBUG [Thread-2481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:03,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:03,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:03,059 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-24T02:49:03,059 INFO [Thread-2479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2479. 2024-11-24T02:49:03,060 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:03,060 DEBUG [Thread-2479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T02:49:03,060 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,060 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,060 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,060 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:03,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:03,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112445d5755d0ab64cf9ae2db3b6f0e3fae5_824d61e10dc42fc1ca4048b972dd9659 is 50, key is test_row_0/A:col10/1732416543058/Put/seqid=0 2024-11-24T02:49:03,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742466_1642 (size=12454) 2024-11-24T02:49:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-24T02:49:03,177 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3801 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3595 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3543 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3783 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3591 2024-11-24T02:49:03,177 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T02:49:03,178 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:49:03,178 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:49:03,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:03,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:03,179 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T02:49:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T02:49:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-24T02:49:03,182 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416543182"}]},"ts":"1732416543182"} 2024-11-24T02:49:03,184 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T02:49:03,184 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T02:49:03,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T02:49:03,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, UNASSIGN}] 2024-11-24T02:49:03,187 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, UNASSIGN 2024-11-24T02:49:03,188 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=CLOSING, regionLocation=7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:03,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, UNASSIGN because future has completed 2024-11-24T02:49:03,190 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T02:49:03,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827}] 2024-11-24T02:49:03,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-24T02:49:03,344 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(122): Close 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:03,344 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T02:49:03,344 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1722): Closing 824d61e10dc42fc1ca4048b972dd9659, disabling compactions & flushes 2024-11-24T02:49:03,344 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:03,467 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:49:03,472 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112445d5755d0ab64cf9ae2db3b6f0e3fae5_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112445d5755d0ab64cf9ae2db3b6f0e3fae5_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:03,473 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/54ab9de673c147fab969ad792142c0f4, store: [table=TestAcidGuarantees family=A region=824d61e10dc42fc1ca4048b972dd9659] 2024-11-24T02:49:03,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/54ab9de673c147fab969ad792142c0f4 is 175, key is test_row_0/A:col10/1732416543058/Put/seqid=0 2024-11-24T02:49:03,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742467_1643 (size=31255) 2024-11-24T02:49:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-24T02:49:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-24T02:49:03,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/54ab9de673c147fab969ad792142c0f4 2024-11-24T02:49:03,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/fdc103008bde4d0a8a119a4347b7a365 is 50, key is test_row_0/B:col10/1732416543058/Put/seqid=0 2024-11-24T02:49:03,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742468_1644 (size=12301) 2024-11-24T02:49:04,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/fdc103008bde4d0a8a119a4347b7a365 2024-11-24T02:49:04,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/c77ca334ab2b432596ac3099ad9052e3 is 50, key is test_row_0/C:col10/1732416543058/Put/seqid=0 2024-11-24T02:49:04,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742469_1645 (size=12301) 2024-11-24T02:49:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-24T02:49:04,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/c77ca334ab2b432596ac3099ad9052e3 2024-11-24T02:49:04,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/A/54ab9de673c147fab969ad792142c0f4 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/54ab9de673c147fab969ad792142c0f4 2024-11-24T02:49:04,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/54ab9de673c147fab969ad792142c0f4, entries=150, sequenceid=330, filesize=30.5 K 2024-11-24T02:49:04,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/B/fdc103008bde4d0a8a119a4347b7a365 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/fdc103008bde4d0a8a119a4347b7a365 2024-11-24T02:49:04,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/fdc103008bde4d0a8a119a4347b7a365, entries=150, sequenceid=330, filesize=12.0 K 2024-11-24T02:49:04,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/.tmp/C/c77ca334ab2b432596ac3099ad9052e3 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c77ca334ab2b432596ac3099ad9052e3 2024-11-24T02:49:04,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c77ca334ab2b432596ac3099ad9052e3, entries=150, sequenceid=330, filesize=12.0 K 2024-11-24T02:49:04,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 824d61e10dc42fc1ca4048b972dd9659 in 1685ms, sequenceid=330, compaction requested=true 2024-11-24T02:49:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 824d61e10dc42fc1ca4048b972dd9659: 2024-11-24T02:49:04,743 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T02:49:04,743 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:04,743 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. because compaction request was cancelled 2024-11-24T02:49:04,743 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. after waiting 0 ms 2024-11-24T02:49:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:49:04,743 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:A 2024-11-24T02:49:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:04,743 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:04,743 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. because compaction request was cancelled 2024-11-24T02:49:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 824d61e10dc42fc1ca4048b972dd9659:C, priority=-2147483648, current under compaction store size is 2 2024-11-24T02:49:04,743 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. because compaction request was cancelled 2024-11-24T02:49:04,743 DEBUG [RS:0;7c69a60bd8f6:39461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:B 2024-11-24T02:49:04,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T02:49:04,743 DEBUG [RS:0;7c69a60bd8f6:39461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 824d61e10dc42fc1ca4048b972dd9659:C 2024-11-24T02:49:04,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/e8a690457e524d428afacdf6f4ee83e6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/f3e965dafdf642029a846f032e76978f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5f9b7eea9d71450faddd3c70e32d8775, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/d0a0da70ae34449292f15ee336dae92a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/c495ff67843a4bdaa018ea6988d631b4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/2b9326feaa9c4fd58a0a8145e587c823, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/4357555ea64947f9bd63279a36487a73, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/545e4cad89e3437bb3d4879d04b30bd8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9048161e71ce4b3c954b30e771e30f06, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/8e19cb74c1c34345b03674ab373237c2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5c1aaa6b864646e5a822e9eee3ae34f4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9fe4615aced04299a1d7d195b31922a4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5ab650af56eb4979b228b3d6336b1d95, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0d2b8cf86a31439995c9e44393a1de6d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/04b3f8cf6ff94bd3ac41d5cfa329aaba, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/814e3100b4854bafba72cc7cc4ea4c3a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/b8f7cf857b64492bb08d03ba297b3652, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/1727d4693e574b0c9227b7770e306a99, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/80fb9933922b45fa87605683c8bfea8c] to archive 2024-11-24T02:49:04,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:49:04,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/e8a690457e524d428afacdf6f4ee83e6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/e8a690457e524d428afacdf6f4ee83e6 2024-11-24T02:49:04,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/f3e965dafdf642029a846f032e76978f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/f3e965dafdf642029a846f032e76978f 2024-11-24T02:49:04,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5f9b7eea9d71450faddd3c70e32d8775 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5f9b7eea9d71450faddd3c70e32d8775 2024-11-24T02:49:04,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/d0a0da70ae34449292f15ee336dae92a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/d0a0da70ae34449292f15ee336dae92a 2024-11-24T02:49:04,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/c495ff67843a4bdaa018ea6988d631b4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/c495ff67843a4bdaa018ea6988d631b4 2024-11-24T02:49:04,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/2b9326feaa9c4fd58a0a8145e587c823 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/2b9326feaa9c4fd58a0a8145e587c823 2024-11-24T02:49:04,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/4357555ea64947f9bd63279a36487a73 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/4357555ea64947f9bd63279a36487a73 2024-11-24T02:49:04,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/545e4cad89e3437bb3d4879d04b30bd8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/545e4cad89e3437bb3d4879d04b30bd8 2024-11-24T02:49:04,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9048161e71ce4b3c954b30e771e30f06 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9048161e71ce4b3c954b30e771e30f06 2024-11-24T02:49:04,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/8e19cb74c1c34345b03674ab373237c2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/8e19cb74c1c34345b03674ab373237c2 2024-11-24T02:49:04,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5c1aaa6b864646e5a822e9eee3ae34f4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5c1aaa6b864646e5a822e9eee3ae34f4 2024-11-24T02:49:04,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9fe4615aced04299a1d7d195b31922a4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/9fe4615aced04299a1d7d195b31922a4 2024-11-24T02:49:04,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5ab650af56eb4979b228b3d6336b1d95 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/5ab650af56eb4979b228b3d6336b1d95 2024-11-24T02:49:04,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0d2b8cf86a31439995c9e44393a1de6d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0d2b8cf86a31439995c9e44393a1de6d 2024-11-24T02:49:04,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/04b3f8cf6ff94bd3ac41d5cfa329aaba to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/04b3f8cf6ff94bd3ac41d5cfa329aaba 2024-11-24T02:49:04,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/814e3100b4854bafba72cc7cc4ea4c3a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/814e3100b4854bafba72cc7cc4ea4c3a 2024-11-24T02:49:04,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/b8f7cf857b64492bb08d03ba297b3652 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/b8f7cf857b64492bb08d03ba297b3652 2024-11-24T02:49:04,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/1727d4693e574b0c9227b7770e306a99 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/1727d4693e574b0c9227b7770e306a99 2024-11-24T02:49:04,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/80fb9933922b45fa87605683c8bfea8c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/80fb9933922b45fa87605683c8bfea8c 2024-11-24T02:49:04,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/1409c002dc85494b96e55656a738d9b6, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/9f2ecd0e17ef45e6be11a7b0430e9525, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/326392c656094f19813a6a18a6878ece, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/5e799dbba36048909454bd5515806e62, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/996b4b73be0d4d9a860587644d4d3c5c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/d6a0dc327dc44554948a5930de4977ce, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/08dbd0829bbb4b73aa5ead61a9529369, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/daba5f1de96446c49fa7c37c5682a13d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/b3d9cda38f86415b8c73abc80b319719, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/4d26593138a441a1804360f03986df83, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/41bedafd8cc1439eb332df3ed39ea91e, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3dc841662e034adabb82777348d95fac, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/271f0e10b93145bc919bf06e09528463, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3e006a87b37647c398ca03b445e2545d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/67ee5b66638b4e428ff65bab227134d4, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/74fbccedf6eb4719918ff400d914d7d8, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/6807a225df014467928c2960ab77766d, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/8b78c34da51f4f2198cfaac9de35fcbf, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/910a83ae1d38438da385e45bba5004d5] to archive 2024-11-24T02:49:04,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:49:04,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/1409c002dc85494b96e55656a738d9b6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/1409c002dc85494b96e55656a738d9b6 2024-11-24T02:49:04,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/9f2ecd0e17ef45e6be11a7b0430e9525 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/9f2ecd0e17ef45e6be11a7b0430e9525 2024-11-24T02:49:04,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/326392c656094f19813a6a18a6878ece to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/326392c656094f19813a6a18a6878ece 2024-11-24T02:49:04,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/5e799dbba36048909454bd5515806e62 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/5e799dbba36048909454bd5515806e62 2024-11-24T02:49:04,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/996b4b73be0d4d9a860587644d4d3c5c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/996b4b73be0d4d9a860587644d4d3c5c 2024-11-24T02:49:04,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/d6a0dc327dc44554948a5930de4977ce to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/d6a0dc327dc44554948a5930de4977ce 2024-11-24T02:49:04,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/08dbd0829bbb4b73aa5ead61a9529369 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/08dbd0829bbb4b73aa5ead61a9529369 2024-11-24T02:49:04,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/daba5f1de96446c49fa7c37c5682a13d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/daba5f1de96446c49fa7c37c5682a13d 2024-11-24T02:49:04,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/b3d9cda38f86415b8c73abc80b319719 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/b3d9cda38f86415b8c73abc80b319719 2024-11-24T02:49:04,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/4d26593138a441a1804360f03986df83 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/4d26593138a441a1804360f03986df83 2024-11-24T02:49:04,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/41bedafd8cc1439eb332df3ed39ea91e to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/41bedafd8cc1439eb332df3ed39ea91e 2024-11-24T02:49:04,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3dc841662e034adabb82777348d95fac to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3dc841662e034adabb82777348d95fac 2024-11-24T02:49:04,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/271f0e10b93145bc919bf06e09528463 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/271f0e10b93145bc919bf06e09528463 2024-11-24T02:49:04,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3e006a87b37647c398ca03b445e2545d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/3e006a87b37647c398ca03b445e2545d 2024-11-24T02:49:04,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/67ee5b66638b4e428ff65bab227134d4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/67ee5b66638b4e428ff65bab227134d4 2024-11-24T02:49:04,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/74fbccedf6eb4719918ff400d914d7d8 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/74fbccedf6eb4719918ff400d914d7d8 2024-11-24T02:49:04,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/6807a225df014467928c2960ab77766d to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/6807a225df014467928c2960ab77766d 2024-11-24T02:49:04,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/8b78c34da51f4f2198cfaac9de35fcbf to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/8b78c34da51f4f2198cfaac9de35fcbf 2024-11-24T02:49:04,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/910a83ae1d38438da385e45bba5004d5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/910a83ae1d38438da385e45bba5004d5 2024-11-24T02:49:04,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/617198e0abc548b38c4c4188c3cfedac, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/b8685ee6ae494fd4bf6ce31d6e488ba2, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/70a2b929952e426e97d9892a9f79ea56, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/ee846bc2bd29489baea4319802155304, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/0fed06312a034e18be067b31c6f314a9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/bce4fb2589ff4f50bf2302407e105699, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/21a966444d594860893133d17f15672c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/82803e35f54b42dc964f143f0f9a760b, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/3a6befabe0294df59f48ce6677931db5, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/473ec0a24f7a4d7f99d19437fd229672, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/601af336068c4287854dd3e01e1910c9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/1388e60844064fde8c6c02d3533d883c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/d7a0e5f8c5034cb9bacfefcd6d552489, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e5b1d01bc2ae4167bed9de8306c3029a, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/f02cf4f581044bec8be5e415d96d7c2c, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/dbb34ad6fc9648e78e842ce973cf3b8f, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c46ade6861c04b2996dc6894d8debc66, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/4e93a9d4d33c45ff832206d8d76ff9c9, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/98a068962e7045ddb6393815bf13a43a] to archive 2024-11-24T02:49:04,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T02:49:04,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/617198e0abc548b38c4c4188c3cfedac to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/617198e0abc548b38c4c4188c3cfedac 2024-11-24T02:49:04,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/b8685ee6ae494fd4bf6ce31d6e488ba2 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/b8685ee6ae494fd4bf6ce31d6e488ba2 2024-11-24T02:49:04,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/70a2b929952e426e97d9892a9f79ea56 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/70a2b929952e426e97d9892a9f79ea56 2024-11-24T02:49:04,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/ee846bc2bd29489baea4319802155304 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/ee846bc2bd29489baea4319802155304 2024-11-24T02:49:04,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/0fed06312a034e18be067b31c6f314a9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/0fed06312a034e18be067b31c6f314a9 2024-11-24T02:49:04,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/bce4fb2589ff4f50bf2302407e105699 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/bce4fb2589ff4f50bf2302407e105699 2024-11-24T02:49:04,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/21a966444d594860893133d17f15672c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/21a966444d594860893133d17f15672c 2024-11-24T02:49:04,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/82803e35f54b42dc964f143f0f9a760b to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/82803e35f54b42dc964f143f0f9a760b 2024-11-24T02:49:04,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/3a6befabe0294df59f48ce6677931db5 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/3a6befabe0294df59f48ce6677931db5 2024-11-24T02:49:04,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/473ec0a24f7a4d7f99d19437fd229672 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/473ec0a24f7a4d7f99d19437fd229672 2024-11-24T02:49:04,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/601af336068c4287854dd3e01e1910c9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/601af336068c4287854dd3e01e1910c9 2024-11-24T02:49:04,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/1388e60844064fde8c6c02d3533d883c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/1388e60844064fde8c6c02d3533d883c 2024-11-24T02:49:04,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/d7a0e5f8c5034cb9bacfefcd6d552489 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/d7a0e5f8c5034cb9bacfefcd6d552489 2024-11-24T02:49:04,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e5b1d01bc2ae4167bed9de8306c3029a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e5b1d01bc2ae4167bed9de8306c3029a 2024-11-24T02:49:04,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/f02cf4f581044bec8be5e415d96d7c2c to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/f02cf4f581044bec8be5e415d96d7c2c 2024-11-24T02:49:04,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/dbb34ad6fc9648e78e842ce973cf3b8f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/dbb34ad6fc9648e78e842ce973cf3b8f 2024-11-24T02:49:04,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c46ade6861c04b2996dc6894d8debc66 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c46ade6861c04b2996dc6894d8debc66 2024-11-24T02:49:04,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/4e93a9d4d33c45ff832206d8d76ff9c9 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/4e93a9d4d33c45ff832206d8d76ff9c9 2024-11-24T02:49:04,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/98a068962e7045ddb6393815bf13a43a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/98a068962e7045ddb6393815bf13a43a 2024-11-24T02:49:04,799 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/recovered.edits/333.seqid, newMaxSeqId=333, maxSeqId=4 2024-11-24T02:49:04,799 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659. 2024-11-24T02:49:04,799 DEBUG [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1676): Region close journal for 824d61e10dc42fc1ca4048b972dd9659: Waiting for close lock at 1732416543344Running coprocessor pre-close hooks at 1732416543344Disabling compacts and flushes for region at 1732416543344Disabling writes for close at 1732416544743 (+1399 ms)Writing region close event to WAL at 1732416544796 (+53 ms)Running coprocessor post-close hooks at 1732416544799 (+3 ms)Closed at 1732416544799 2024-11-24T02:49:04,801 INFO [RS_CLOSE_REGION-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(157): Closed 824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:04,801 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=824d61e10dc42fc1ca4048b972dd9659, regionState=CLOSED 2024-11-24T02:49:04,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 because future has completed 2024-11-24T02:49:04,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-24T02:49:04,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseRegionProcedure 824d61e10dc42fc1ca4048b972dd9659, server=7c69a60bd8f6,39461,1732416369827 in 1.6130 sec 2024-11-24T02:49:04,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=178, resume processing ppid=177 2024-11-24T02:49:04,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, ppid=177, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=824d61e10dc42fc1ca4048b972dd9659, UNASSIGN in 1.6180 sec 2024-11-24T02:49:04,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-24T02:49:04,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6220 sec 2024-11-24T02:49:04,808 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732416544807"}]},"ts":"1732416544807"} 2024-11-24T02:49:04,808 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T02:49:04,809 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T02:49:04,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6300 sec 2024-11-24T02:49:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-24T02:49:05,338 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:49:05,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T02:49:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:49:05,342 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:49:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-24T02:49:05,342 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=180, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:49:05,346 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,348 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C, FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/recovered.edits] 2024-11-24T02:49:05,349 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0110e087e88d43baa4a4c728c9fc910f to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/0110e087e88d43baa4a4c728c9fc910f 2024-11-24T02:49:05,350 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/54ab9de673c147fab969ad792142c0f4 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/54ab9de673c147fab969ad792142c0f4 2024-11-24T02:49:05,351 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/7c4e4cf6b7c74814ad2698f0e19684c6 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/A/7c4e4cf6b7c74814ad2698f0e19684c6 2024-11-24T02:49:05,353 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/61d45a070e524ce48d7fa9566d3e8e21 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/61d45a070e524ce48d7fa9566d3e8e21 2024-11-24T02:49:05,353 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/88d72de7cc2c450d9c095f24a028eb85 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/88d72de7cc2c450d9c095f24a028eb85 2024-11-24T02:49:05,354 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/fdc103008bde4d0a8a119a4347b7a365 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/B/fdc103008bde4d0a8a119a4347b7a365 2024-11-24T02:49:05,356 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/727c9ab9224540129cab23e42a3ae1f3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/727c9ab9224540129cab23e42a3ae1f3 2024-11-24T02:49:05,357 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c77ca334ab2b432596ac3099ad9052e3 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/c77ca334ab2b432596ac3099ad9052e3 2024-11-24T02:49:05,357 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e4c2c823a346408bb3de0e3b533f170a to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/C/e4c2c823a346408bb3de0e3b533f170a 2024-11-24T02:49:05,359 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/recovered.edits/333.seqid to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659/recovered.edits/333.seqid 2024-11-24T02:49:05,359 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/default/TestAcidGuarantees/824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,359 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T02:49:05,360 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:49:05,360 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T02:49:05,362 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124286f60b1ca954d7f9e5bbe3a0350f5c8_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124286f60b1ca954d7f9e5bbe3a0350f5c8_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,363 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242fcb19e0af5f458285c6f056485ba1cd_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242fcb19e0af5f458285c6f056485ba1cd_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,364 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112434fad882d07a455aa543dc45d53c77c5_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112434fad882d07a455aa543dc45d53c77c5_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,366 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243cf5cf63bb5746fca314cf3b00826a9b_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243cf5cf63bb5746fca314cf3b00826a9b_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,367 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112445d5755d0ab64cf9ae2db3b6f0e3fae5_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112445d5755d0ab64cf9ae2db3b6f0e3fae5_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,368 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245b74465eb04b47328164ebd547d08165_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245b74465eb04b47328164ebd547d08165_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,369 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112465cc347de65f45fe842aec9191421ce0_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112465cc347de65f45fe842aec9191421ce0_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,370 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124769ed39f2a3e48a48ad53a8fe9adfaeb_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124769ed39f2a3e48a48ad53a8fe9adfaeb_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,371 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112480be143cd8c94f9fb7a98d6170bf1862_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112480be143cd8c94f9fb7a98d6170bf1862_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,372 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249873efe5075049f284e9dc91afdc3212_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249873efe5075049f284e9dc91afdc3212_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,373 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249ef83ab6a7ac4efd866ccb214f662d4c_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249ef83ab6a7ac4efd866ccb214f662d4c_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,374 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aaa203be9e114413b1c5360f3cd39e29_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aaa203be9e114413b1c5360f3cd39e29_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,376 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2fa877cc1d142c980608089a7ec036a_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124c2fa877cc1d142c980608089a7ec036a_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,379 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e38bdd122a0949d39ce07002a011fc5c_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e38bdd122a0949d39ce07002a011fc5c_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,380 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4270a7e3f91478bae1e778f82989b1c_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4270a7e3f91478bae1e778f82989b1c_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,382 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ea23365559f642bfb1c46ac117f2d270_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124ea23365559f642bfb1c46ac117f2d270_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,383 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f7779e8581d04b01b6f1d0b6a9a73eaf_824d61e10dc42fc1ca4048b972dd9659 to hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f7779e8581d04b01b6f1d0b6a9a73eaf_824d61e10dc42fc1ca4048b972dd9659 2024-11-24T02:49:05,384 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T02:49:05,387 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=180, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:49:05,389 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T02:49:05,392 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T02:49:05,393 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=180, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:49:05,393 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T02:49:05,393 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732416545393"}]},"ts":"9223372036854775807"} 2024-11-24T02:49:05,395 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T02:49:05,395 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 824d61e10dc42fc1ca4048b972dd9659, NAME => 'TestAcidGuarantees,,1732416519515.824d61e10dc42fc1ca4048b972dd9659.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T02:49:05,395 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T02:49:05,395 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732416545395"}]},"ts":"9223372036854775807"} 2024-11-24T02:49:05,397 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T02:49:05,397 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=180, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T02:49:05,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-11-24T02:49:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=43749 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-24T02:49:05,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T02:49:05,447 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T02:49:05,456 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 241) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 516), ProcessCount=11 (was 11), AvailableMemoryMB=9818 (was 9919) 2024-11-24T02:49:05,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T02:49:05,457 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:49:05,457 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:49:05,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:05,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:05,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T02:49:05,457 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T02:49:05,457 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-24T02:49:05,457 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c69a60bd8f6,43749,1732416369043 2024-11-24T02:49:05,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T02:49:05,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:49:05,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T02:49:05,489 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T02:49:05,489 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:49:05,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T02:49:05,490 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:49:05,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:05,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:49:05,490 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c69a60bd8f6,39461,1732416369827' ***** 2024-11-24T02:49:05,490 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T02:49:05,490 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T02:49:05,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T02:49:05,490 INFO [RS:0;7c69a60bd8f6:39461 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T02:49:05,491 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(959): stopping server 7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c69a60bd8f6:39461. 2024-11-24T02:49:05,491 DEBUG [RS:0;7c69a60bd8f6:39461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T02:49:05,491 DEBUG [RS:0;7c69a60bd8f6:39461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T02:49:05,491 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T02:49:05,492 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T02:49:05,492 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T02:49:05,492 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:49:05,492 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T02:49:05,492 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T02:49:05,492 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T02:49:05,492 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T02:49:05,492 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T02:49:05,492 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-24T02:49:05,517 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/info/760965ea5c50455e80d4f6b940893c99 is 95, key is TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730./info:/1732416400534/DeleteFamily/seqid=0 2024-11-24T02:49:05,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742470_1646 (size=6514) 2024-11-24T02:49:05,575 INFO [regionserver/7c69a60bd8f6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T02:49:05,641 INFO [regionserver/7c69a60bd8f6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T02:49:05,641 INFO [regionserver/7c69a60bd8f6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T02:49:05,692 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:49:05,893 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:49:05,921 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/info/760965ea5c50455e80d4f6b940893c99 2024-11-24T02:49:05,947 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/ns/f528ea977e9d4a1d9a2c3a0a4938ca8b is 93, key is TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730./ns:/1732416400497/DeleteFamily/seqid=0 2024-11-24T02:49:05,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742471_1647 (size=5906) 2024-11-24T02:49:06,093 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:49:06,293 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:49:06,351 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/ns/f528ea977e9d4a1d9a2c3a0a4938ca8b 2024-11-24T02:49:06,369 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/rep_barrier/cc6318eb1eaf4e80876c1d07fd17a53e is 102, key is TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730./rep_barrier:/1732416400497/DeleteFamily/seqid=0 2024-11-24T02:49:06,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742472_1648 (size=6025) 2024-11-24T02:49:06,493 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:49:06,693 DEBUG [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T02:49:06,772 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/rep_barrier/cc6318eb1eaf4e80876c1d07fd17a53e 2024-11-24T02:49:06,796 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/table/2ee7d25adff847b5abd3d6ca08218e60 is 96, key is TestAcidGuarantees,,1732416374471.129703e293b1a78847241fc258ac8730./table:/1732416400497/DeleteFamily/seqid=0 2024-11-24T02:49:06,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742473_1649 (size=5936) 2024-11-24T02:49:06,799 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/table/2ee7d25adff847b5abd3d6ca08218e60 2024-11-24T02:49:06,803 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/info/760965ea5c50455e80d4f6b940893c99 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/info/760965ea5c50455e80d4f6b940893c99 2024-11-24T02:49:06,807 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/info/760965ea5c50455e80d4f6b940893c99, entries=12, sequenceid=90, filesize=6.4 K 2024-11-24T02:49:06,808 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/ns/f528ea977e9d4a1d9a2c3a0a4938ca8b as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/ns/f528ea977e9d4a1d9a2c3a0a4938ca8b 2024-11-24T02:49:06,812 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/ns/f528ea977e9d4a1d9a2c3a0a4938ca8b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-24T02:49:06,813 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/rep_barrier/cc6318eb1eaf4e80876c1d07fd17a53e as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/rep_barrier/cc6318eb1eaf4e80876c1d07fd17a53e 2024-11-24T02:49:06,817 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/rep_barrier/cc6318eb1eaf4e80876c1d07fd17a53e, entries=6, sequenceid=90, filesize=5.9 K 2024-11-24T02:49:06,818 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/.tmp/table/2ee7d25adff847b5abd3d6ca08218e60 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/table/2ee7d25adff847b5abd3d6ca08218e60 2024-11-24T02:49:06,822 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/table/2ee7d25adff847b5abd3d6ca08218e60, entries=7, sequenceid=90, filesize=5.8 K 2024-11-24T02:49:06,823 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1331ms, sequenceid=90, compaction requested=false 2024-11-24T02:49:06,827 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-24T02:49:06,828 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T02:49:06,828 INFO [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T02:49:06,828 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732416545492Running coprocessor pre-close hooks at 1732416545492Disabling compacts and flushes for region at 1732416545492Disabling writes for close at 1732416545492Obtaining lock to block concurrent updates at 1732416545492Preparing flush snapshotting stores in 1588230740 at 1732416545492Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732416545493 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732416545493Flushing 1588230740/info: creating writer at 1732416545493Flushing 1588230740/info: appending metadata at 1732416545517 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732416545517Flushing 1588230740/ns: creating writer at 1732416545929 (+412 ms)Flushing 1588230740/ns: appending metadata at 1732416545947 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732416545947Flushing 1588230740/rep_barrier: creating writer at 1732416546354 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732416546368 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732416546368Flushing 1588230740/table: creating writer at 1732416546775 (+407 ms)Flushing 1588230740/table: appending metadata at 1732416546795 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732416546795Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3712cd8: reopening flushed file at 1732416546803 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51715da1: reopening flushed file at 1732416546808 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fb16328: reopening flushed file at 1732416546812 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@636e8e08: reopening flushed file at 1732416546817 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1331ms, sequenceid=90, compaction requested=false at 1732416546823 (+6 ms)Writing region close event to WAL at 1732416546824 (+1 ms)Running coprocessor post-close hooks at 1732416546828 (+4 ms)Closed at 1732416546828 2024-11-24T02:49:06,828 DEBUG [RS_CLOSE_META-regionserver/7c69a60bd8f6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T02:49:06,894 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(976): stopping server 7c69a60bd8f6,39461,1732416369827; all regions closed. 2024-11-24T02:49:06,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741834_1010 (size=25344) 2024-11-24T02:49:06,901 DEBUG [RS:0;7c69a60bd8f6:39461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/oldWALs 2024-11-24T02:49:06,902 INFO [RS:0;7c69a60bd8f6:39461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7c69a60bd8f6%2C39461%2C1732416369827.meta:.meta(num 1732416373844) 2024-11-24T02:49:06,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741832_1008 (size=15191815) 2024-11-24T02:49:06,911 DEBUG [RS:0;7c69a60bd8f6:39461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/oldWALs 2024-11-24T02:49:06,911 INFO [RS:0;7c69a60bd8f6:39461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 7c69a60bd8f6%2C39461%2C1732416369827:(num 1732416372794) 2024-11-24T02:49:06,911 DEBUG [RS:0;7c69a60bd8f6:39461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T02:49:06,911 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T02:49:06,911 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T02:49:06,911 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.ChoreService(370): Chore service for: regionserver/7c69a60bd8f6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T02:49:06,912 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T02:49:06,912 INFO [regionserver/7c69a60bd8f6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T02:49:06,912 INFO [RS:0;7c69a60bd8f6:39461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39461 2024-11-24T02:49:06,962 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c69a60bd8f6,39461,1732416369827 2024-11-24T02:49:06,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T02:49:06,962 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T02:49:06,973 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c69a60bd8f6,39461,1732416369827] 2024-11-24T02:49:06,983 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c69a60bd8f6,39461,1732416369827 already deleted, retry=false 2024-11-24T02:49:06,983 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c69a60bd8f6,39461,1732416369827 expired; onlineServers=0 2024-11-24T02:49:06,984 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c69a60bd8f6,43749,1732416369043' ***** 2024-11-24T02:49:06,984 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T02:49:06,984 INFO [M:0;7c69a60bd8f6:43749 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T02:49:06,984 INFO [M:0;7c69a60bd8f6:43749 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T02:49:06,984 DEBUG [M:0;7c69a60bd8f6:43749 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T02:49:06,984 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T02:49:06,984 DEBUG [M:0;7c69a60bd8f6:43749 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T02:49:06,984 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.large.0-1732416372490 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.large.0-1732416372490,5,FailOnTimeoutGroup] 2024-11-24T02:49:06,984 INFO [M:0;7c69a60bd8f6:43749 {}] hbase.ChoreService(370): Chore service for: master/7c69a60bd8f6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T02:49:06,985 INFO [M:0;7c69a60bd8f6:43749 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T02:49:06,985 DEBUG [M:0;7c69a60bd8f6:43749 {}] master.HMaster(1795): Stopping service threads 2024-11-24T02:49:06,985 INFO [M:0;7c69a60bd8f6:43749 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T02:49:06,985 DEBUG [master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.small.0-1732416372491 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c69a60bd8f6:0:becomeActiveMaster-HFileCleaner.small.0-1732416372491,5,FailOnTimeoutGroup] 2024-11-24T02:49:06,985 INFO [M:0;7c69a60bd8f6:43749 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T02:49:06,985 ERROR [M:0;7c69a60bd8f6:43749 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:40051 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40051,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-24T02:49:06,986 INFO [M:0;7c69a60bd8f6:43749 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T02:49:06,986 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T02:49:06,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T02:49:06,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T02:49:06,994 DEBUG [M:0;7c69a60bd8f6:43749 {}] zookeeper.ZKUtil(347): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T02:49:06,994 WARN [M:0;7c69a60bd8f6:43749 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T02:49:06,995 INFO [M:0;7c69a60bd8f6:43749 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/.lastflushedseqids 2024-11-24T02:49:07,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742474_1650 (size=72) 2024-11-24T02:49:07,073 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:49:07,073 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39461-0x1016abbe9c70001, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:49:07,074 INFO [RS:0;7c69a60bd8f6:39461 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T02:49:07,074 INFO [RS:0;7c69a60bd8f6:39461 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c69a60bd8f6,39461,1732416369827; zookeeper connection closed. 2024-11-24T02:49:07,074 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@25a2ee22 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@25a2ee22 2024-11-24T02:49:07,075 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T02:49:07,406 INFO [M:0;7c69a60bd8f6:43749 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T02:49:07,407 INFO [M:0;7c69a60bd8f6:43749 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T02:49:07,407 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T02:49:07,407 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:49:07,407 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:49:07,407 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T02:49:07,407 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:49:07,407 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=773.05 KB heapSize=950.53 KB 2024-11-24T02:49:07,427 DEBUG [M:0;7c69a60bd8f6:43749 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1609227da64546a3a394df23357b7eb8 is 82, key is hbase:meta,,1/info:regioninfo/1732416373960/Put/seqid=0 2024-11-24T02:49:07,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742475_1651 (size=5672) 2024-11-24T02:49:07,831 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1609227da64546a3a394df23357b7eb8 2024-11-24T02:49:07,858 DEBUG [M:0;7c69a60bd8f6:43749 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/625c859e1efc4499869c01f99ab917d5 is 2276, key is \x00\x00\x00\x00\x00\x00\x00^/proc:d/1732416466287/Put/seqid=0 2024-11-24T02:49:07,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742476_1652 (size=45958) 2024-11-24T02:49:08,106 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T02:49:08,262 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=772.50 KB at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/625c859e1efc4499869c01f99ab917d5 2024-11-24T02:49:08,267 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 625c859e1efc4499869c01f99ab917d5 2024-11-24T02:49:08,288 DEBUG [M:0;7c69a60bd8f6:43749 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2c6850fc44c3403fa9249e0fec1cd641 is 69, key is 7c69a60bd8f6,39461,1732416369827/rs:state/1732416372550/Put/seqid=0 2024-11-24T02:49:08,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073742477_1653 (size=5156) 2024-11-24T02:49:08,691 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2186 (bloomFilter=true), to=hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2c6850fc44c3403fa9249e0fec1cd641 2024-11-24T02:49:08,695 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1609227da64546a3a394df23357b7eb8 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1609227da64546a3a394df23357b7eb8 2024-11-24T02:49:08,698 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1609227da64546a3a394df23357b7eb8, entries=8, sequenceid=2186, filesize=5.5 K 2024-11-24T02:49:08,699 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/625c859e1efc4499869c01f99ab917d5 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/625c859e1efc4499869c01f99ab917d5 2024-11-24T02:49:08,702 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 625c859e1efc4499869c01f99ab917d5 2024-11-24T02:49:08,702 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/625c859e1efc4499869c01f99ab917d5, entries=180, sequenceid=2186, filesize=44.9 K 2024-11-24T02:49:08,703 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2c6850fc44c3403fa9249e0fec1cd641 as hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2c6850fc44c3403fa9249e0fec1cd641 2024-11-24T02:49:08,705 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40051/user/jenkins/test-data/d25f7ebe-8c34-9ea5-5ecf-acc4377801a3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2c6850fc44c3403fa9249e0fec1cd641, entries=1, sequenceid=2186, filesize=5.0 K 2024-11-24T02:49:08,706 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(3140): Finished flush of dataSize ~773.05 KB/791606, heapSize ~950.23 KB/973040, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2186, compaction requested=false 2024-11-24T02:49:08,707 INFO [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T02:49:08,707 DEBUG [M:0;7c69a60bd8f6:43749 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732416547407Disabling compacts and flushes for region at 1732416547407Disabling writes for close at 1732416547407Obtaining lock to block concurrent updates at 1732416547407Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732416547407Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=791606, getHeapSize=973280, getOffHeapSize=0, getCellsCount=2391 at 1732416547407Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732416547408 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732416547408Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732416547427 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732416547427Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732416547834 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732416547857 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732416547857Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732416548267 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732416548287 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732416548287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b7a8e7: reopening flushed file at 1732416548694 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5262b171: reopening flushed file at 1732416548698 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b0ad443: reopening flushed file at 1732416548702 (+4 ms)Finished flush of dataSize ~773.05 KB/791606, heapSize ~950.23 KB/973040, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1299ms, sequenceid=2186, compaction requested=false at 1732416548706 (+4 ms)Writing region close event to WAL at 1732416548707 (+1 ms)Closed at 1732416548707 2024-11-24T02:49:08,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36769 is added to blk_1073741830_1006 (size=934935) 2024-11-24T02:49:08,710 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T02:49:08,710 INFO [M:0;7c69a60bd8f6:43749 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T02:49:08,710 INFO [M:0;7c69a60bd8f6:43749 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43749 2024-11-24T02:49:08,711 INFO [M:0;7c69a60bd8f6:43749 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T02:49:08,841 INFO [M:0;7c69a60bd8f6:43749 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T02:49:08,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:49:08,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43749-0x1016abbe9c70000, quorum=127.0.0.1:55039, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T02:49:08,846 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T02:49:08,849 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:49:08,849 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:49:08,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T02:49:08,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/hadoop.log.dir/,STOPPED} 2024-11-24T02:49:08,852 WARN [BP-79769874-172.17.0.2-1732416365305 heartbeating to localhost/127.0.0.1:40051 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T02:49:08,852 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T02:49:08,852 WARN [BP-79769874-172.17.0.2-1732416365305 heartbeating to localhost/127.0.0.1:40051 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-79769874-172.17.0.2-1732416365305 (Datanode Uuid ef9ddf6b-c708-403b-baaa-8ef179148346) service to localhost/127.0.0.1:40051 2024-11-24T02:49:08,852 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T02:49:08,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545/data/data1/current/BP-79769874-172.17.0.2-1732416365305 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T02:49:08,855 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/cluster_2fcb9e0d-0ce1-2dca-4dcf-9a046afef545/data/data2/current/BP-79769874-172.17.0.2-1732416365305 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T02:49:08,856 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T02:49:08,863 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T02:49:08,864 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T02:49:08,864 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T02:49:08,864 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T02:49:08,864 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a81a37bf-bff0-dd2f-c453-de77413c089a/hadoop.log.dir/,STOPPED} 2024-11-24T02:49:08,884 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T02:49:09,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down